{ "suite_sha256": "cd444ee142dc6b87e74967c9925ea9b7b171b90fe0e18b5a214c4587d964121a", "runs": [ { "id": "20261025T170652Z", "startedAt": "2026-10-25T17:06:52.146Z", "harnessVersion": "0.1.0", "harnessSha256": "ad9b5de9fa2c9608a2aedd94a51e1b8532d243b03e6789c70ee626b3c8a538b4", "suiteSha256": "cd444ee142dc6b87e74967c9925ea9b7b171b90fe0e18b5a214c4587d964121a", "settings": { "quality_max_tokens": 4000, "temperature": 0, "reasoning_effort": "low" }, "proofFiles": 80, "manifestSha256Recorded": "f2ad278156cfb48af8071c16ccdd638ea301e2630db37c847725e13cbfb12c99", "manifestFilesRecorded": 80, "manifestSha256OnDisk": "f2ad278156cfb48af8071c16ccdd638ea301e2630db37c847725e13cbfb12c99", "manifestFilesOnDisk": 80 }, { "id": "20261025T172733Z", "startedAt": "2026-10-25T17:56:53.343Z", "harnessVersion": "0.1.0", "harnessSha256": "daea9ded7865329aef3f0f39b08051af366966de716753201105df25d1171c8f", "suiteSha256": "cd444ee142dc6b87e74967c9925ea9b7b171b90fe0e18b5a214c4587d964121a", "settings": { "quality_max_tokens": 4000, "temperature": 0, "reasoning_effort": "low" }, "proofFiles": 1280, "manifestSha256Recorded": "3666ce9a9f0b0f1bca7c216bbd95741233dad3d24e76aedcb906024c5f84d92f", "manifestFilesRecorded": 1280, "manifestSha256OnDisk": "3666ce9a9f0b0f1bca7c216bbd95741233dad3d24e76aedcb906024c5f84d92f", "manifestFilesOnDisk": 1280 }, { "id": "20261025T195000Z", "startedAt": "2026-10-26T03:44:50.952Z", "harnessVersion": "0.1.0", "harnessSha256": "daea9ded7865329aef3f0f39b08051af366966de716753201105df25d1171c8f", "suiteSha256": "cd444ee142dc6b87e74967c9925ea9b7b171b90fe0e18b5a214c4587d964121a", "settings": { "quality_max_tokens": 4000, "temperature": 0, "reasoning_effort": "low" }, "proofFiles": 320, "manifestSha256Recorded": "7516a93b12563ec4ea0822987195e76a2ff9791f7ebd8dfecf71aa88f43e2a75", "manifestFilesRecorded": 320, "manifestSha256OnDisk": "7516a93b12563ec4ea0822987195e76a2ff9791f7ebd8dfecf71aa88f43e2a75", "manifestFilesOnDisk": 320 }, { "id": "20261025T200000Z", "startedAt": "2026-10-26T05:22:44.688Z", "harnessVersion": "0.1.0", "harnessSha256": "daea9ded7865329aef3f0f39b08051af366966de716753201105df25d1171c8f", "suiteSha256": "cd444ee142dc6b87e74967c9925ea9b7b171b90fe0e18b5a214c4587d964121a", "settings": { "quality_max_tokens": 4000, "temperature": 0, "reasoning_effort": "low" }, "proofFiles": 2160, "manifestSha256Recorded": "ca68d5a31ce63896c540958f18d89994e99b53d0ad9d4cd9b5f3b3e07bd4aa5e", "manifestFilesRecorded": 2160, "manifestSha256OnDisk": "ca68d5a31ce63896c540958f18d89994e99b53d0ad9d4cd9b5f3b3e07bd4aa5e", "manifestFilesOnDisk": 2160 }, { "id": "20261027T100730Z", "startedAt": "2026-10-27T10:07:30.431Z", "harnessVersion": "0.1.0", "harnessSha256": "6d522eab4cfc8cdb4e18d42fbeef8d7040b83cd3d661100aea022a03321d4625", "suiteSha256": "cd444ee142dc6b87e74967c9925ea9b7b171b90fe0e18b5a214c4587d964121a", "settings": { "quality_max_tokens": 4000, "temperature": 0, "reasoning_effort": "low" }, "proofFiles": 160, "manifestSha256Recorded": "e59ab17eecb6bc7266215fa241088b4336c2829bd14272c8f258a2057d140407", "manifestFilesRecorded": 166, "manifestSha256OnDisk": "e59ab17eecb6bc7266215fa241088b4336c2829bd14272c8f258a2057d140407", "manifestFilesOnDisk": 166 }, { "id": "20261027T103825Z", "startedAt": "2026-10-27T17:30:24.045Z", "harnessVersion": "0.1.0", "harnessSha256": "6d522eab4cfc8cdb4e18d42fbeef8d7040b83cd3d661100aea022a03321d4625", "suiteSha256": "cd444ee142dc6b87e74967c9925ea9b7b171b90fe0e18b5a214c4587d964121a", "settings": { "quality_max_tokens": 4000, "temperature": 0, "reasoning_effort": "low" }, "proofFiles": 560, "manifestSha256Recorded": "43797d279243f0105e95a0a96d24fdf6fb44983d344a684b384d380d2b6df946", "manifestFilesRecorded": 560, "manifestSha256OnDisk": "43797d279243f0105e95a0a96d24fdf6fb44983d344a684b384d380d2b6df946", "manifestFilesOnDisk": 560 }, { "id": "20261027T120517Z", "startedAt": "2026-10-27T17:31:31.658Z", "harnessVersion": "0.1.0", "harnessSha256": "6d522eab4cfc8cdb4e18d42fbeef8d7040b83cd3d661100aea022a03321d4625", "suiteSha256": "cd444ee142dc6b87e74967c9925ea9b7b171b90fe0e18b5a214c4587d964121a", "settings": { "quality_max_tokens": 4000, "temperature": 0, "reasoning_effort": "low" }, "proofFiles": 1200, "manifestSha256Recorded": "9695c140f6c40cc1a4b4c040c50a82061fe20fb83056f50cf7f5f7b6d9851958", "manifestFilesRecorded": 1200, "manifestSha256OnDisk": "7173b3caaf3e4b2352aba8ef4027b017dc25e25e46c1d2b30e14d43b66994241", "manifestFilesOnDisk": 1600 } ], "items": [ { "model": "thinkingmachines/inkling", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793102717-ejcBboruBLemcTQXx7Zq", "proof_sha256": "de5fa622f361e67a2f6d5da725a174cf6ff92b36c8207374b0ed3386dcdc60fd" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793102742-CztrNwHhCBoMRbTOsama", "proof_sha256": "006be56021cbb8c4a2b87f2af77e22f631135a686ac335bf17f4b4eb533401d0" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793102782-hpl6ZWsrrJHBBU1LhNJ7", "proof_sha256": "e97439ed58b7eecb8114e60e39247e914a630f44031f6e69881c89e05456f85e" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793102821-d7NN2Iy42YWRBpRpHigE", "proof_sha256": "e22ae284ecfd1a5e74992fbbc88874d9fb9963f39d0f7f8e6a20411be30e6bf5" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793102843-C6vnhNZaWgvL1scDU1bx", "proof_sha256": "363a21b6786aca6bfb0bd6147a5c57d9560d5de6db9ac8fe9630b8eedec5d813" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793102885-DVWtMvXdSV8rOox6SOmo", "proof_sha256": "0cbfcee7d3e52fdebb79fb89f69966a6ddb9ebe5c3c7786f7176ad4a653a4024" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793102907-lDeBwqtRyJ96PG58OoWb", "proof_sha256": "ef7794ca6889bf11dbb1061e501e06362c761fe72a225812628ad5bce37450ff" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793102950-LkNk3NTG2YZLhP6QfyQ7", "proof_sha256": "5262c080904fb8e31dfe7ef3bebdc885fb6408d9396a2fcd595ba8dc4452ea70" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793102988-fpJxVR15vCvpwvuTlz6N", "proof_sha256": "0164c279a08adc7e6fa09fc293a3421c7bcb0a3a1da71417fd13fee963c4e774" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793102999-mCISH5A1lB5Ov6FEtJVQ", "proof_sha256": "5c47902736130184b695052384077fb78a46e4a2b402b135978c0280340373a5" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103038-fPXT4k7saHGcHdfs4JKr", "proof_sha256": "8769d25259469d72119168d481e5c6adfc0773c0a76f301408a5b5ae0d2bd22f" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103078-RMt8LsKuWpcu2GHzrKcB", "proof_sha256": "a915e264b1e888ff0beba8c4f539a2133aa059d5a14b13b41d5291e8a4565991" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103116-e2yaGoOITuPcu9vnx9UG", "proof_sha256": "7fce89d87c5676f552bf9bf3fca61aee2c72692f5764c634e86fe070c9335078" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103156-Ga3jbMsWyJSKcByFJlJr", "proof_sha256": "7d28d5115c12002ae50381d06c2386ac8e5cd340902cb30d6c0b6c3e411e0a7a" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103170-LBO9t9XyZYVTdDpV8Dol", "proof_sha256": "8af86ecd5fedb6ba6f48ce8605d15298ec88ffe280be9a4dd12fb27e7a9e1e2a" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103211-Q9NeKBadegSIiyR5H1xh", "proof_sha256": "217876a349b803d0e2f951d112d5e8447c4bcb10edd2b7478d1b3d0a083074e5" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103228-nyNy9bgQsK0DT9Cx8Zup", "proof_sha256": "b880511aace2544da69fd6859ca7eec0548de55e4ef5d33c86d1da249fefd1b2" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103267-WI7h925XzzApYk4Dbh7b", "proof_sha256": "a212122007d9f9ac43b2a6dd5059d0de4713961a2507a69b7ae25b1ddf6b7e9f" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103302-iTdp5ELFu4W91pSSVKy1", "proof_sha256": "2a9d590112a45e8fe59cc747ec380453a50ac4a38758009f30c62888357c8e2b" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103322-SlqakJsEDyqTgvXVcLXT", "proof_sha256": "105a9f6480eb01ec978ebe5f72823c8e4538f2a7cd799a035696a9721051e8c4" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103354-E2j2eJ6xFblUiCxScAJ5", "proof_sha256": "5df0a5d3b62dc56d2f5f1e8145afeaf4968c9b95d88b6f863944edaad0ad93d1" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103378-9X8JiyzdIaJ4JvAdvNvr", "proof_sha256": "18afef751b58d12bc138d99e0f937cddc5e78ec42bb2ff1f71d8cc587cbe4d3b" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103410-afJ0jIT3LzACgtuJvWYf", "proof_sha256": "4e0cb53277bae03f589d68f5afa93ba0eccda3b097f8289993dd727d9b3c737f" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103427-S3ZKpm27DKTdmdLaKXam", "proof_sha256": "30ad25720863c8b358b6177351758974a9c62f50e09f57b9fe95347337c02043" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103444-wVrdNTLFlysdyHIYkA6b", "proof_sha256": "d530e0d8455e4b924560bc08a2aa958e627b1511251aa4bbeb47dd625d2d6901" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103451-Uzba6lGonMzIPoRu0sgm", "proof_sha256": "ff94d587173e6a9952e4ee5ce115fe246ee10cff0dc1a0aeab50843a760cec55" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103458-dJspIAmDkKarK1CCrTLP", "proof_sha256": "e8de5771de2b342f38d94a05bba012244c09a37488eee48c93ef080c7b819599" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103477-k40oIhiIa3NvZ9gOJvks", "proof_sha256": "d315d37d5581a4ecea48e06bc9d99b8408ab3b132fb31ef1435de0be61258861" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103494-DRCM1Hr4KjKEvkWzfCRK", "proof_sha256": "26140f0fac0a1c0113e89854548a8b380d0973ea7aadc8e2e1a70e9a0d480616" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103510-E79YEnZ8U2fctEbSwtlL", "proof_sha256": "cb0ecdc726b1710a8b3f5c4baf650938c08476cdf337e27c8f9338d841f89bf4" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103532-XWOJUhhQ3E03iPEev1dy", "proof_sha256": "48223c553ab19ead483f5fafab9fde1fd5df4cc5057442f340664025012853e0" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103542-gjtLeewNuOQifrLWDZwN", "proof_sha256": "02b1245223ac1fbf6e3bcd69b6831a17073f6620b224c6197efdbe3bb4668487" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103559-O8KDdPIeFckeA8mMv5ZN", "proof_sha256": "598eb653305cb0f444c05df17e8fd5cc68e202303f944c84df73effa24812799" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103566-IbFD6Kc1z55pe20ql6wm", "proof_sha256": "b40ff9afbaefcb8f1c8b5a6d3cdc59156578c3575f674444880e88665c69506d" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103583-QPoH2lGJq3hMa73izbNc", "proof_sha256": "feac7258f3d8651b1dd18fa0f7294b6237cdd95bc69927d6446a4ccb41574325" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103590-GNjWYE4V90FTO6VIX2SK", "proof_sha256": "1a1a3134f324e42d5c9af2057fc6710868407765e1a07163c232ed6a9582e7a5" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103608-hdf3en6QbKKD2ZiiD4ar", "proof_sha256": "7c03e33f278e00891c15814bf04413985a8cfc7a20dc936f0d347eb77fd5a357" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103638-vuTZJRAAOLoivfAjGS8A", "proof_sha256": "f16160f59bff3168a8f706bbf90fab9db53141381f5ecc882a69f467ada17e7e" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103650-iY9t03UxGvRQSKy8jCJ6", "proof_sha256": "92ae558544bc1cc8be6edda0e8d4dcf88697abcbb9bc9c5517cf80769a772de5" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103689-gKJEN0vpfLKKTHYbooxV", "proof_sha256": "1056e8d12bdcdbebfb1c38bd8d634d4978ad51b70c8afe9368072fb58b77a425" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103727-VvYJVwmUg5gg7Wk1QLDc", "proof_sha256": "bdbdd3f9bb8acd0aa47d050758cb477a490fbaf2bc0940d6267f9d137b5eb35b" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103737-SohDSGqZSeB0bSmvIHZb", "proof_sha256": "87f6a32fc6cabd009bef98a53ae90b1ae70c76393a62f50166298f0007a21c95" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103754-TU2Kjf26tHmdxZPHR8bL", "proof_sha256": "95a2b19d806897c1ab840fbfe9840b80ee4b26c70baee637589e346e9f9f0cf0" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103761-5dS4g1Dq5a1SUp0RCoLZ", "proof_sha256": "ea75b78b8cf5f68afcfc62569723dc99e9bf04d8c0893aea6bcd7f592058af47" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103800-c59dJ2hspNL416kZrYGp", "proof_sha256": "d048986fc4773d9088f59eceef871e8c775f090addb0be07b8120e0f1e0b3199" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103840-pEomVo3Ud3au9uo52Xvt", "proof_sha256": "88508139982d48cf7b1347aafa87371dd4713d73fdbccc408111d90957b804e3" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103848-o2JV0AcFdZkAmdEMypTw", "proof_sha256": "a91e05991c837e88329c0f9d448666bb3cb635a3255d8c79bbfb8efaa6d9c1dd" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103855-3OK6aydNxrzU3OT0yLJG", "proof_sha256": "214970fd75f82d675ef148d258702c428e5f505cf7c2c1a19156a60230a4bcc0" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793104324-Iz3R94AOs2aUvYDt1Kdf", "proof_sha256": "f69fc16a42923ec26e5523c62ab3af964706cbf905015dc34df86a1757029c48" }, { "model": "thinkingmachines/inkling", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793105235-x2Cip4nxBPi9lm0M3oY1", "proof_sha256": "4f22b13a2e5019126b9a69f3f93112112aa2d5ea8e11f61552db21029e969ba3" }, { "model": "thinkingmachines/inkling", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793105276-F35I5DNqcZKjpm4IZZm0", "proof_sha256": "2931b6ff72bb2828f7e3f3e3e4cb4207313a4e9d7fd36098a867e5317245f70a" }, { "model": "thinkingmachines/inkling", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793106197-FhwnXkodHRBnVPJCMPQr", "proof_sha256": "8f25e2e788f6b4e78c867f83a43bc9d8da916f14cf3faef31acf94615b965e2d" }, { "model": "thinkingmachines/inkling", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793107113-DohhDD298Fm738qhDFSr", "proof_sha256": "7da113da0945cd635058e6da02ce60f61ff3b1b4c201866a7facfa9e5b8b5e6a" }, { "model": "thinkingmachines/inkling", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793107322-sayFYqMZuFDULbCJEH6T", "proof_sha256": "c7fe3344f381e99d0705452ed71838b1b9b8b82dea79b72d1ed4f62d9c497b1c" }, { "model": "thinkingmachines/inkling", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793108233-wndfTgtsYKDbFn23OWh1", "proof_sha256": "cbfa623954ee4a6747fe841f057c609ae151dcfc3bcb3ce144adac8edddabfc2" }, { "model": "thinkingmachines/inkling", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793108272-AUqe2WNXd5cW7aHd3cbk", "proof_sha256": "d4ab768a1ef15c255b6dd227a65ca41b0d7463b0fbe412d74c02b9c80e98a015" }, { "model": "thinkingmachines/inkling", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793108289-iDApkmJ7K0UNTyhxDnW5", "proof_sha256": "9daa56badd107d9a9e684f1873300e4565b64fbabf737be73eaed663d0e2c233" }, { "model": "thinkingmachines/inkling", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793109194-UMxxi9gkiwHZuJ4FdQmx", "proof_sha256": "cd2e85ce5dd3e346c15d8eeac11178f5f1bd2a9abfdd9b994e44a4ec64fa1e19" }, { "model": "thinkingmachines/inkling", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793110107-kJUytAL6j1Vo0knymsfP", "proof_sha256": "1ab14599a412218e669d66e0f07b5d5d39cc680c3ca6b95d16a8542faa55c88b" }, { "model": "thinkingmachines/inkling", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793110124-1H1n1TSnhxOuQc1WG7y1", "proof_sha256": "7c30f3713efb9a7d3be60c0b25f6e62b58f1cd236c1ae8960ecf74bc2f7f6a01" }, { "model": "thinkingmachines/inkling", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793111032-HsM0lKIMXDjtHgZRUqSj", "proof_sha256": "39f686ccfaed479d48ff720bc78388577166fdd1e8a2ffe59d376f05f804f67d" }, { "model": "thinkingmachines/inkling", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793111954-Ab2SM8C5uZl2vkREZDDd", "proof_sha256": "6636b2c2572f4405ebed70bb9b9cbe99060ac1832427964286a6c5eb321e6f05" }, { "model": "thinkingmachines/inkling", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793112893-0kc4jZzFe9I04DA0CMsK", "proof_sha256": "71874fdb3269715795d2576cd44ea36062bff4ef5e8b8c9269443c887b21213d" }, { "model": "thinkingmachines/inkling", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793112910-6qpvvOy4TQCXh95RUTz6", "proof_sha256": "036ced484967d0c06771d74b80c27de1ba8089dae27c9d66757e35a4157ae30e" }, { "model": "thinkingmachines/inkling", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793112918-90JsW1NDPmIlkGaihPvL", "proof_sha256": "3bde4a113e5223e0cc1c9e506a82da84c1a53de19ef454ffbc74044fd982e12c" }, { "model": "thinkingmachines/inkling", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793112926-2JWoViS7ohtvUrWs4cwI", "proof_sha256": "2c9755ef089ed4efd89dfe9ca0ec2fc46f4dd558aad3171290268b050e0831b0" }, { "model": "thinkingmachines/inkling", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793113838-Y4VXHx2li6VIaHBRt1oI", "proof_sha256": "dc30248f76a2a02842f8a36ca98e7e18e5519fddf927c8f575f1a9a03b94329f" }, { "model": "thinkingmachines/inkling", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793113855-v52i5Mt5kP7bXKB5XsGK", "proof_sha256": "1e104d1ff115bfbd9445a13db084f9ee450f5959c465b20b721237b0adc49e92" }, { "model": "thinkingmachines/inkling", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793113874-RBhktXRPSks7YcGSau7X", "proof_sha256": "063cad2836cce2eb8759ba7c2b865ec9a8f57cde0eb96735f799f27bcface353" }, { "model": "thinkingmachines/inkling", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793113892-VhBo5yLGrrmickNaqT6o", "proof_sha256": "0e85feba89342a2699bd81153592d3c4720c1b67cf592e4918a6ac0822ba4bea" }, { "model": "thinkingmachines/inkling", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793113900-hNSrjvCFzzR5xNWlWDEc", "proof_sha256": "d7330a51cefc3a45fad9548ab04ff109980b625c7799bd470b65a4737d528e45" }, { "model": "thinkingmachines/inkling", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793113907-4vte7AQOEZ2fwZhA4JEl", "proof_sha256": "65e1821fc2b11070cf6e0fc62e9f0010e122e6fed29a1fc3c023f87e65cda1e2" }, { "model": "thinkingmachines/inkling", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793113915-a9lHFWJCvIDAEjsLYfDp", "proof_sha256": "14a01bb17bc6f7ca527086b6d9c170082dfd4fb9f2465e25c4f666181998e28d" }, { "model": "thinkingmachines/inkling", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793113932-QScmmfnfH0UsMcDuw1yx", "proof_sha256": "04076ec7523fdb51a1f83b258e3cd57415a0069a05fe2782562b4d1155764738" }, { "model": "thinkingmachines/inkling", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793113941-NL0B174i1800dJAPRJ2l", "proof_sha256": "144bca4997cf9cdb5d188804a1c4f80cd1e9efedb719aacafacd5337eea0dfb8" }, { "model": "thinkingmachines/inkling", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793113959-k1uOnu5zgRHKeTjgzhQ4", "proof_sha256": "e54c0b387c50e30a4fafc7309998a200833976599eef35365f62d9786be9c0c3" }, { "model": "thinkingmachines/inkling", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793113976-wxdhkkN0JNWXyQX2hdJ4", "proof_sha256": "7627debf75c19591c51e5f8cbc950218ab8ae25660dbd6fe6569e1c360b3c128" }, { "model": "thinkingmachines/inkling", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793113994-f4pNQdZXn85gFhoRtZ5L", "proof_sha256": "10779e2690c92e93936b3d7f25fa9b98338657509af247130009f28bf59958c7" }, { "model": "thinkingmachines/inkling", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793114002-LqH8JSM3i8myU9U9ndRr", "proof_sha256": "abfd0b802b3fe9ede3b85c992f3341427951122a36c498d0427faca5163f8c21" }, { "model": "thinkingmachines/inkling", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793114009-wywDUrI15pl6niXYhFYp", "proof_sha256": "9da16a2b879f965eb7b235933c194e5e0878f5af9736fa4fbe1b70fbac09ca7b" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793102717-0O17RL5YeQGfsTNA0Jce", "proof_sha256": "b9447f66be8cb7b701d14f5312f2f5c4c23a4d1e4b0115b85c03625c669e0f65" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793102737-QoH5aSWu5OSAQBeLNgHp", "proof_sha256": "2ced26c720e0cac80693e242f90fec8fc94f93c4d46f64cf029b3f3b2c3c050e" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261027T120517Z", "generation_id": "gen-1793102763-Midz68IjocPSxB8wCHfw", "proof_sha256": "9e24a727c79a1cfa683908a6a2771cdfd17f59370ae5c5bcbf0665c042092e0c" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "StepFun", "run_id": "20261027T120517Z", "generation_id": "gen-1793102797-jmDk4pJmTfRGZXEZWait", "proof_sha256": "d6755c55cf222cea6eea7c92d84af4b7514ff0a4e77ab65c148a10c9a0c86fb4" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "StepFun", "run_id": "20261027T120517Z", "generation_id": "gen-1793102849-921HgjoBSiTtRPsG2Mhh", "proof_sha256": "da3acc5d532181a3dd50cfe4033c0a5163dd72e603f4010d754372cd3e0e0a41" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793102884-jWI7ZTsSR1AFw2iiCeSV", "proof_sha256": "0ffa8047fc6e1e0a0b9027225a783d2b48950013aacdccb91a98e07afd7ccfb3" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "StepFun", "run_id": "20261027T120517Z", "generation_id": "gen-1793102926-xcGbVOUxMx8D6aaQeODL", "proof_sha256": "e5e305e2b17eb5d0eca0ea1401fbc479d66fdbc0431d2ca53e56d1f2720fb68c" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793102968-2flc9Dch8fEEtkusoSlZ", "proof_sha256": "25fff39c92886f01d2973a25295e9e0e785cf9a6c537fcac4933bfecb723edf6" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793103009-DnBfhmJZWRDxVFnftols", "proof_sha256": "6652ddd5450cd0bfe0a16e347457b7b8ad9a2505657fe8024c19180e357e5c28" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "InferenceNet", "run_id": "20261027T120517Z", "generation_id": "gen-1793103056-ikND5TWRprIiQgBQazVb", "proof_sha256": "7e130a346d17fe321ba117ca9758bb3cf31e35b91c1f8f24e02f892aee44587f" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Featherless", "run_id": "20261027T120517Z", "generation_id": "gen-1793103110-tWwpqCiKoJN2x6lam6Bv", "proof_sha256": "047bc3b30ed0a4d7c09bdf485b6395103bd033d2519e49b41b0310bf5007f450" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "InferenceNet", "run_id": "20261027T120517Z", "generation_id": "gen-1793103159-wFHT1hd3Ekfj7VXPS16a", "proof_sha256": "5fe6194f015cfaa24cc1f88e5d918a5bea33b405cbba147e1a4fc4a4cdd0728c" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Featherless", "run_id": "20261027T120517Z", "generation_id": "gen-1793103217-scObXaxBBK4SvlyEUfT5", "proof_sha256": "632262187d2ee06acc4842b48b2cbf79df351e640b08bb7002651476b0d369b5" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "StepFun", "run_id": "20261027T120517Z", "generation_id": "gen-1793103229-zzbghtETAFMpF0UzopR3", "proof_sha256": "41aaccfac89601ec09c16cbd40b1739fe094c1620ed1166f4bb5ccee624e4db0" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261027T120517Z", "generation_id": "gen-1793103271-wuTAVCsmz9p3ycYYfIwF", "proof_sha256": "ca5fedf872c739baf44f68226fdcc55803406e94e46d81611f775d05887d5476" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793103299-vqv4qjgwzrrJiStLZjRp", "proof_sha256": "71a1703abbceb35b4dc47c6dd1ee70c3f7a1e4f8025a69092b2154ccb3d81e6b" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Featherless", "run_id": "20261027T120517Z", "generation_id": "gen-1793103319-lP6gyzPUHpr6rKszrZQc", "proof_sha256": "2acc85899aebd46046d399112b064f198e0cb84e136de7e456f0cc7f14d53419" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "StepFun", "run_id": "20261027T120517Z", "generation_id": "gen-1793103380-MH5sZR8EeErkIM12TK7G", "proof_sha256": "089f22baf184b0d54562bb37bd50dcaa21f742fc71c64e4e5cc322a3670f63aa" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "StepFun", "run_id": "20261027T120517Z", "generation_id": "gen-1793103410-6WTY7Rkvo7jzwIky8kYw", "proof_sha256": "909d69e1a95e2f8d65fe76d41ae2db845f34676f9fb1ee695eb2545e22a0dcb6" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261027T120517Z", "generation_id": "gen-1793103432-ZKaNpks7jp6cAk7YKGGi", "proof_sha256": "16732689c37dbde47a98994075081e8898302929dfa41fe9012d23f8eac37c78" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261027T120517Z", "generation_id": "gen-1793103440-8L0ONqZuCifqTVEV3cGi", "proof_sha256": "cc2f45b9fca3873764c9bee438648908294d2af41fdd9e9d7ca8abc37d303e98" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793103450-eUp95hCNvBeHg6RA2mux", "proof_sha256": "60824b4269e7cd79d8aa6271dfb30365a49cbc1591d8769530c5b4e55a193a17" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "InferenceNet", "run_id": "20261027T120517Z", "generation_id": "gen-1793103461-bVrn2mSIHcjGDnYHBjKH", "proof_sha256": "02aa251e40b6413d6fb8e93eb2317d44d13f563be104008c83c5316fe1b6d03b" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793103497-kXkVSzD39wCddtWsogYI", "proof_sha256": "f1278a63a9aa830e0d45be0067689b35eaa0259a09cf4f57b9e137435732aec4" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "StepFun", "run_id": "20261027T120517Z", "generation_id": "gen-1793103515-52LbJx70aggWdDoM5beC", "proof_sha256": "d466f26e8ba7d91354e440d68e7ef038eaa0f2e6e4f8f7708dfc87ac0085e656" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "InferenceNet", "run_id": "20261027T120517Z", "generation_id": "gen-1793103540-wwxsTyUqTmpIM1gugmNF", "proof_sha256": "21b684f22bb8d93750cfab91beac85001c7bf916f4cd0dc1a0a07226daef01a4" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261027T120517Z", "generation_id": "gen-1793103568-EFC7AjeHVPeINwB2eycf", "proof_sha256": "0c44d742982d9e2d929c70685a0f621f67815ef580880a2758be101fd52d4b46" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "StepFun", "run_id": "20261027T120517Z", "generation_id": "gen-1793103581-09jJ9F0BqETaWr0LdO4b", "proof_sha256": "1afd18cf7c513b476986b6f33ec2de7fdba8e53f064cbe94199f5c7284468d84" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793103614-3dPMZ3pNdWcrLDslNDVt", "proof_sha256": "fbf88bac442478711e004edc34028ec5b2669a09ae88b0b001af05e4ca165c48" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "InferenceNet", "run_id": "20261027T120517Z", "generation_id": "gen-1793103631-yJkaoscLA8rd48DL6uNd", "proof_sha256": "a338c985c428b3584dbd9230ead87822fd22a797229ce83b238607340d43a006" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Featherless", "run_id": "20261027T120517Z", "generation_id": "gen-1793103662-bihbPu9XH5KIrmH1fiW5", "proof_sha256": "0445343a5aa00acbe903f80cbc6b2dbd0866555a7bdecb01b44f3863f515af50" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "InferenceNet", "run_id": "20261027T120517Z", "generation_id": "gen-1793103749-UrqLoOtIowDvWWnMERHx", "proof_sha256": "596d6f5bc668e73fd7f6546c290867f550053df0591b42ebdd26d4e5e3bb0203" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261027T120517Z", "generation_id": "gen-1793103775-MaqwJY6R4SOhCrRULaR1", "proof_sha256": "a4a8fea45faca0d2d68b6c9dc4ee02bd2443a87a62777182eaf5f8cd07131db5" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "InferenceNet", "run_id": "20261027T120517Z", "generation_id": "gen-1793103788-0tRTmucoKNN7YCCMsgCs", "proof_sha256": "04c4e5047c78deacb98e8d8cb887cc6f2bb3079a665bef719831bc469a677ea7" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "StepFun", "run_id": "20261027T120517Z", "generation_id": "gen-1793103817-Em2Qo3opYQQuQxiwM7GI", "proof_sha256": "580975930d33d7b4cff7ffdfd192fe0ec7271f0bc0b9452f2b7a8c6d9f8132d1" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793103840-4k86vBJLoWLIWyVtMkjI", "proof_sha256": "6daf2179fa01a1735661333928cd4a35388e0a38669585b2f04d9602bff54b92" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261027T120517Z", "generation_id": "gen-1793103854-TR43HM4cLsyzAjLdw5zd", "proof_sha256": "13d837d4c311217f139a194621189c07e8b8672aa05283bcc62ce58a8343039a" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Featherless", "run_id": "20261027T120517Z", "generation_id": "gen-1793104323-r8RegwD4cHNbe46coQz7", "proof_sha256": "aa5bfe0fdf64f03c7450d34793b867c809516a9ca6771721e962685694831c8b" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Featherless", "run_id": "20261027T120517Z", "generation_id": "gen-1793105237-icc5SN9UBJoxUOg360li", "proof_sha256": "9885d380c8bdae684c57e96ad0c9483b31b2e79d27f40b2f07c49d8b0a79a7db" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261027T120517Z", "generation_id": "gen-1793108245-uoN5foop76NkjtQpEfxa", "proof_sha256": "dfbc448bee48002eb71faadf48c326ebed0537a74acd3d2af9a3cdf2ce2a1bbf" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793113912-kqSoJEl7Wwgdj4i66SIz", "proof_sha256": "e2000f83c732ac6aaa6e2b7f2afa3a9eedc7dbe6d70edb5bdbc570d563ea85f0" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261027T120517Z", "generation_id": "gen-1793113944-KHuEcBATFRbxFRad69ep", "proof_sha256": "d12816d67d74d34fb9f91e5651de98513c9fd210c9f63f93ec10bf60aeb39b97" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "InferenceNet", "run_id": "20261027T120517Z", "generation_id": "gen-1793113963-cMX707PlrhRdzrKAvIC8", "proof_sha256": "bf44b05ccd5578f126f0e0969f028b0a7afbd2b2de2e96b5ab8b83f21f33ab7d" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "InferenceNet", "run_id": "20261027T120517Z", "generation_id": "gen-1793113986-d3cjVFu83AQzOZYI1etc", "proof_sha256": "10ebf402856f16a34dac6edbbc8836376fc693857a6fec3031f7239bda5ea673" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261027T120517Z", "generation_id": "gen-1793114010-OPyOTq2jUWtYGL7h5nVv", "proof_sha256": "25da28fdc9c490fe5b3461a203eebfbfc0c0eb46da575280572a7382a0450224" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261027T120517Z", "generation_id": "gen-1793114027-Ejt7tJhaJsaqRQtQcOOV", "proof_sha256": "eba8097b1b5407c002b14ae2cac5f84119e7906c299d71b9389902eabde0bdc8" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793114045-IHdHhgWcBPFs5RQBgO2m", "proof_sha256": "79e6590fca846816f2d1292d91644c0509eda3531d0e469ef94fbb4e1c426951" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793114063-oos12VfNWe57isMtmfde", "proof_sha256": "a945bf4180b24e6b2a3aab45562b2a34d691cc1d55be1c4f61718035e82d6a79" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Featherless", "run_id": "20261027T120517Z", "generation_id": "gen-1793114080-tz2E4hKSlsCGenQIem6L", "proof_sha256": "bb9a601537b7f2b6b95c76e1d934cff780710f56584210ef7938da05a0e05a7a" }, { "model": "stepfun/step-5-preview", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Featherless", "run_id": "20261027T120517Z", "generation_id": "gen-1793114175-nkQuRz5uDvbHT1dpFXI4", "proof_sha256": "5286810ce4cf29e2a2eb7e748ebea5d7e2d766aaf39759c5189a20b267c186a9" }, { "model": "stepfun/step-5-preview", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "InferenceNet", "run_id": "20261027T120517Z", "generation_id": "gen-1793114202-6ctvDKzpAqxYXTOAwlcC", "proof_sha256": "151fa1216e7c4f2980fd6e66c7bace6cb78b8a8425b0c8c192b10d9b5d09d2b0" }, { "model": "stepfun/step-5-preview", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261027T120517Z", "generation_id": "gen-1793114252-tFnrX1LjozCuJnFgiMUn", "proof_sha256": "148bab052cf39a0c7fd83d019dfda4085ac92acb3c10bc05be2377c763ce233a" }, { "model": "stepfun/step-5-preview", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261027T120517Z", "generation_id": "gen-1793114270-3tibbMcDi2XlH8LLnpsa", "proof_sha256": "c041b108fce2fdb1d7c95823256684a799c20dd4e106a5ac701ae959e4cf09c5" }, { "model": "stepfun/step-5-preview", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "StepFun", "run_id": "20261027T120517Z", "generation_id": "gen-1793114289-JHGYQ6rbcUpWLflSANeh", "proof_sha256": "b8054be9e4dc58b1050b75bfdb2b973cc30df555e5f03dbf95e09f7a997d5c8e" }, { "model": "stepfun/step-5-preview", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793114409-JkAb7lhNELJ7sHoF7oYU", "proof_sha256": "2cebbc1528f24af3ae0f3478706d55d11912893dbd84e71f795efc0084b7c3c8" }, { "model": "stepfun/step-5-preview", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793114432-I9vdnhucFrTd1eIzGTTX", "proof_sha256": "3b27e0e877b3a907107dba5d4c6f72d655b205f94d5fb352428f484e9c72e875" }, { "model": "stepfun/step-5-preview", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793114447-4zpCGwGuMB9ty5luqr4A", "proof_sha256": "f9daf38df20d255f67df4018718b11b3e74b9ba48735ab575a1aa38b8e697c76" }, { "model": "stepfun/step-5-preview", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261027T120517Z", "generation_id": "gen-1793114458-ZSTbxNKFjWz9JhQd7i9m", "proof_sha256": "eb9b9521dffcdb9f1bf8a4666fb0018dfca2bf36d6ad4b71be296b4b91457ac6" }, { "model": "stepfun/step-5-preview", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261027T120517Z", "generation_id": "gen-1793114476-5jmG9f4zpN9wrfMUicPM", "proof_sha256": "115b6a53f66c8e0447f2df9281d57859f845a745aceb42a8d039b06e20912e9e" }, { "model": "stepfun/step-5-preview", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793114485-mngPcNWlGN28CX3lEVL6", "proof_sha256": "46e14d9e94b98d7666cd58881eccf24fa0fc600db3cd986bfdfb969564d9de4e" }, { "model": "stepfun/step-5-preview", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "StepFun", "run_id": "20261027T120517Z", "generation_id": "gen-1793114494-cVXCqTZce3Xd43yHssey", "proof_sha256": "096e40e679109ea5c50f78e70d54e3b524b164c9adc6a31f4b72da710e355853" }, { "model": "stepfun/step-5-preview", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Featherless", "run_id": "20261027T120517Z", "generation_id": "gen-1793114513-FkDhSkqU5tdWZxdz1iIh", "proof_sha256": "ed05a08ed37a506756b52907c74438dd764fdc78357a9b1afe350c6365205146" }, { "model": "stepfun/step-5-preview", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "StepFun", "run_id": "20261027T120517Z", "generation_id": "gen-1793114535-aCuVhUcDO6qw6TqpC0bu", "proof_sha256": "79e23f8034fc4414fb38f8ff28cef0d77b26a4bbceb810b765c0065d334ec571" }, { "model": "stepfun/step-5-preview", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261027T120517Z", "generation_id": "gen-1793114553-jVc8gEB7WwnArKyCNPxH", "proof_sha256": "248c0fafddc50ae52fa6ec6bb8e1750c56f42279b383413a5a242c43a057f534" }, { "model": "stepfun/step-5-preview", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261027T120517Z", "generation_id": "gen-1793114571-06Qv7teEdHS2r8fdkujI", "proof_sha256": "e4af48b4e8855f616550398408cffaedb579db8397b15411d83ef1f59f8eb546" }, { "model": "stepfun/step-5-preview", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793114590-FjEaThFfMQEQpqZXtpHv", "proof_sha256": "a6036f202fa757f5ab5f95b5b70f28a5f5985b3c6deb01b8ba86c32f6b1c1cd5" }, { "model": "stepfun/step-5-preview", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Featherless", "run_id": "20261027T120517Z", "generation_id": "gen-1793114607-P1Cf2iVtKvmh75uQOhO7", "proof_sha256": "6028074c5581d7c40d6eed6318bbd428028461a79bd39bab82a9a0ff2148eb1c" }, { "model": "stepfun/step-5-preview", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "StepFun", "run_id": "20261027T120517Z", "generation_id": "gen-1793114618-8bo5ANZcR4ksaTUk02pr", "proof_sha256": "5aa9a99c300738c12512f016099b258dbba6531da9883f983f2862dc8548c009" }, { "model": "stepfun/step-5-preview", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Featherless", "run_id": "20261027T120517Z", "generation_id": "gen-1793114659-3GPvT51MDwBx7A0wMsuZ", "proof_sha256": "1a608a7d75a18c11492aaa8a8e760d1479e1a81470a87521ba8500fb0fd6c228" }, { "model": "stepfun/step-5-preview", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261027T120517Z", "generation_id": "gen-1793114679-niW1L5WuYCQqqXOdKJCI", "proof_sha256": "ee8617e4f74dd34c3ce5ce9b7452ef7a78dc57f388239c5f37b9bc8175cb1134" }, { "model": "stepfun/step-5-preview", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Featherless", "run_id": "20261027T120517Z", "generation_id": "gen-1793114696-xz0wrBPPQ1O02GV8ptqP", "proof_sha256": "ee160ed51b495b693eef9fb6684e6ee65eb335ed908ac7c18f289f31f6197aab" }, { "model": "stepfun/step-5-preview", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793114717-YaAlYw60tFcpGYeQYZ6I", "proof_sha256": "558b80167a387a614b4f08f7eb04a8028f9bdd8a9366ee43fa825517c01114b9" }, { "model": "stepfun/step-5-preview", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793114742-SjxSVjFythRDaebqV0ci", "proof_sha256": "0c4447d57b6d7a55c6f268651f4258a99b9942166ce5dc0b10945aae93067da4" }, { "model": "stepfun/step-5-preview", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261027T120517Z", "generation_id": "gen-1793114760-OpZaRJcYnP1nUDW5YPr6", "proof_sha256": "df61fd71072bd3c227a56ad0ed1872e6e9419e9446ee7bb043f5b54145c67209" }, { "model": "stepfun/step-5-preview", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793114771-X70roiy29NrfNYX2S8Dy", "proof_sha256": "6c43b0492ea6491566fe4cc14dc8d2ae2bd3910c1d8f3801a0a8f4665ac9ff01" }, { "model": "stepfun/step-5-preview", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "StepFun", "run_id": "20261027T120517Z", "generation_id": "gen-1793114790-hxBFFBMLR3kG1TbJ6umj", "proof_sha256": "1a59184bcb2c12e2417ca07bd632b2adf234ce8b7d2423b2cf631e66926f5c2f" }, { "model": "stepfun/step-5-preview", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261027T120517Z", "generation_id": "gen-1793114809-YkW9LR6mGiOxiOMDf7qG", "proof_sha256": "12c94dace44e2657b9ca6a1daf46b0a77284252480d87e4cab5d6e20a94badce" }, { "model": "stepfun/step-5-preview", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "StepFun", "run_id": "20261027T120517Z", "generation_id": "gen-1793114830-pMD1PGZj3gZp9GVYWWZO", "proof_sha256": "6e29b314b2a988c5768df1385e8defe952610bf8cbe0febb0c84c09258f97ed8" }, { "model": "stepfun/step-5-preview", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Featherless", "run_id": "20261027T120517Z", "generation_id": "gen-1793114838-0JHKoiGCRcZNpxBNnroi", "proof_sha256": "c544497f969f5e898d002d3a2b5bd8cbfa9495c978456e56892439924d9bef2b" }, { "model": "stepfun/step-5-preview", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793114858-p3jV2ShfoNld5wq1Ntk5", "proof_sha256": "38e524217250230202c962524f6444de9dd6b3c4988d21781531a533b7ac57cf" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793102718-cynj3pm6zydDyDERtMtF", "proof_sha256": "03e18c777cb64ec3cb99b49c1145ec41d64530325830da26089107a75778ff85" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793102745-d3taAEkHl2kkQCWnAzx9", "proof_sha256": "22d3e2efe94b7c8ddc24a61d64df4a89a0404d3d8f4c7659ab8c056f50acea04" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793102783-rmzbUQgj6wEWHHn3R6bd", "proof_sha256": "2e42cc05dbc2373a2d4a55abdda794a4551af5d764f01ab43d66054e7dcc4120" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793102844-fqc6QoZ6KFQ0SKzMxNof", "proof_sha256": "494d0a1bb5e4b7a0a1b4b07941c3b3447b71c767731da656809a8947108bc62d" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793102882-BHNDaoISJ51tZZLnfqhA", "proof_sha256": "b8d3b3947edc9d2c44e92b69a0e34c2ee2391294cfd0a3cac4e72e7aaa783e0c" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793102910-kqi49rVmQzuKd3DfwcLt", "proof_sha256": "66a53a75cc3e873b5811d124f98782753e6b5c5018f15f83b11bbbecb3648c1c" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793102968-LsYKCxNfbmc1wUVDgzAe", "proof_sha256": "de67119a21ea2969cd36a5a9fe1c7b7685718b028c2dc78963f181fd19463a51" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793103024-n5c25hdhW8pSfBpZpqsu", "proof_sha256": "5a80f5034b8b54269f450d825925b4033be73d232108e3a737bcc81f98f0a88a" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793103085-jwhKhObRoqowUUEAXM2q", "proof_sha256": "bd04278c48973715f8ff9d56f80875f6a153f5ab5e501567f0cc3f37158eadcf" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793103143-0ctPtBuEJGO4ZTyltpuL", "proof_sha256": "d26dfa8c01ff79c1454cbe0d6bc26b5a3c15a6b955d6151db57682538343e0fc" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "substitute back", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793103203-RtEzEBSI7AJ3FWauRVzz", "proof_sha256": "ad62b1b9c62dcb38f63122b6176a942399e77d4c5e0f0380156c8651ffc1906c" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793103262-knqlCSTH5vzPUKQVuMU6", "proof_sha256": "42f3090406fab4d02c59db81d90d2cf2572bedc7452c248c7d2200fac84e4668" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793103322-v5itHvGFlfhlvclUKk6u", "proof_sha256": "e75f629a34a946e0233e937ef25c63eaaa491238cb279931a91928a1120d5508" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793103357-HvF6XLMxVQA7qgpx8V4s", "proof_sha256": "5105850455471d82c33fc6724ca342be036bb0b12fd6aeadf2def92911001f3b" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793103396-zNELBH1N3msKM3ZUKInE", "proof_sha256": "e6462b3c258b3b9d4f41f423348beba36508f22c90d98567985061d07c872884" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793103432-kcWw2V9WQLbaz6DVyBXs", "proof_sha256": "98661a68905b4591f99a1602c1e33fa09ca4ea2d521a0aef67e5f44b83dea1f8" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793103463-c0xHn7pFBQ1zUdBzUjyE", "proof_sha256": "71f0c71205393f539ae7732cf867cd587b6d24f2e5a7953bfa2c70ef2261d4d2" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793103493-X52pLdsAsVzlkllYuhnD", "proof_sha256": "1e0ff933843f698e8d575011f27388f1fa186443bc446ba6d6c4abe87b6673c0" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793103523-VFNhDSUgH640pzMCnL5m", "proof_sha256": "aec1d8a8583683b4dc2e9cbe8f648a465ff88867b5c5c1e5162f98ebae5df61c" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793103551-hq7l0A6s21MY28gKQOy5", "proof_sha256": "50569fa88d2e5c71dbd6184f0f86d37a49b3f0819ccaa8aaf703ec7a7bc5a5d9" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793103570-Rg3sCrvXqPj86llwYk5k", "proof_sha256": "9bdc9a8f9b339fa9fb86e9966bb480db612237e474b0c3b06eb52ae715feaf00" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793103609-y6gNEo9UqXV8R3b0hVP3", "proof_sha256": "6a77d878559e24d7a30425b146f0c07b45c46ff895efd189ff0c79bb80d4f273" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793103647-GZf1AmK12tlvbKzy91xJ", "proof_sha256": "85ebe79ba4bf0d07848feb7a3a2686059508f7ed9077f78838deb932e0ef5fb9" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793103682-e1S04NuTFZoOPrUMU0eJ", "proof_sha256": "0f219e032aff608f3e51f6dbfdee473bfed139eb58b29eee2b8d9cbe01f2409d" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793103742-2p27yCSKqioOXNCNG1iR", "proof_sha256": "bd866e312c06a18759404b36006602def2e5aba0a896baa88c6737cab4618f29" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793103774-h6MPuDLoiLyzziSj9CXZ", "proof_sha256": "96c7ef0a2f1a07ffb6dde3e4e6a3487a87841b4d8fa5a926bc1a2aceab6ad095" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793103814-Ewlkr9FzsOLRs5PTGfpy", "proof_sha256": "a822e8839a3a1a3a320dc0223795025d72d305d7862a01b335b240934d93ca11" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": true, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793122291-xKSrxEw4nelbRtIzzlyi", "proof_sha256": "cc67f9dcf2fceaa36e1e288b5843ff85f2b25342689bd21f81177ba2aa942a14" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793111958-U2R1iAvSueWfxu07uIJl", "proof_sha256": "a91e53993267ac59c10118bce55f5b6bf1dc40831e49176595428237c5db1738" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793112892-N0uZ9sZPVESehzGrXOkv", "proof_sha256": "8b5e831e0bd2a88b946433037e9c7aee3c16bd3ec2b6ca450e2ec8af9f5eef22" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793114018-qiOgwqhDshC4uSrRF4dD", "proof_sha256": "36c84124f0a22a84cbde1f94374fb6e85859f947e75594f9a839c2a1b11847dd" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793114046-cH3Zzr31XxQEJRT9KTmS", "proof_sha256": "93f5287cce5b94e8686636d448e1ec61be745d5dc729faa0b04ac2f7572a9713" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793114075-AUstbRzM0hkVLdR4Uubt", "proof_sha256": "1300161ee3429debbf77060a83f49c0efeded9ddaaaaea22f036fb6d8e7d424d" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793114104-FGv8K14fC1rxT6TABCTS", "proof_sha256": "90447198ec91e6519e430d80127ad3086bff7dd9eedce2aa9d2ee1daa352738f" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793114133-BtgNkW60F9Kh1rMXHIzc", "proof_sha256": "278477925a6c09ff9021591137f2bae0fef9aaf7a16118474396daedc3193fae" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793114171-IMYk1QmFhVyUqXX9pYmn", "proof_sha256": "a619c2f623ddaa78a3cfdcec66fb78e7ebaf78b332932e291de086304da74f00" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793114209-45Phg6kwrkk7BhYvcJ2N", "proof_sha256": "3e46e1483e3e0d1a66e8fe81a2d913dc30311a9a5d6fabb67f39a2114462d7d9" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "- c. csom with cholesteatoma and masto", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793114251-pWJBAXVdZwXN7hZ0jgvi", "proof_sha256": "3e5f88c047e6874e7d44f41c553dc10ba291a74723f2947dcf7c3dedcb6100a0" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "- orbital apex syndrome (h) → a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793114296-bHOAshjkwuIO24T2LF4f", "proof_sha256": "a282a75e1d6573682a241f37143cb9cffd63d54a6015416aa7d7024b76d32023" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793114339-WeLI1xAaLpJdMJ4nSxl8", "proof_sha256": "178e03f384c76a7407f66d31ffac5b65bea875c9d7a23231980895461f48580e" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793114355-NMW9R2buy5QU0PjKoyrm", "proof_sha256": "c5bacde9e83fabcb22ed044ebdeefd4d6b33f0c5d2fd936dbe9e39d1b23c8019" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "among the options, acanthamoeba (b) and acanthamoeba spp. (e) are both plausible. however, in clinical and microbiological contexts, when referring to the causative agent of keratitis", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793114386-3vgTLtYuWv8pCckknneJ", "proof_sha256": "c722a0c75113ba8e546b2b8aa075383979965c4bfb38ce9298436e11a8f2eeed" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793114426-G4xwZsMBtEF2ePd0gGQn", "proof_sha256": "0aa2db6472753155ae188b0094c7af336988cc9b11442853b52932d3bed081ca" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793114466-xI9WwqM3Df1NHkx8Li0l", "proof_sha256": "1d6235324d3da65b837eefe1e684a9ec3c9e4dd2e565541c6fa7b35c1a72edf9" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793114494-UM5VpoPSNt5GLY67BUFl", "proof_sha256": "2c7f4f8db9b100e09c6fafc88fed915d379ce47b76ca1a02fb4128818ed4dc6b" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793114535-Xtnfvh8EDTXeZxI7Rcmr", "proof_sha256": "74aa07f1c65ea424cdcf2d2ba1c96bdbe568ddaab3d22f7acf49ddcea29a7b28" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793114570-P0ZCB1FDiuwopaJJdPnf", "proof_sha256": "e3c10ec89e05f78386e1d5e1cdb6fe448a0f100725b8a70b18fc5b164fab9d46" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793114597-X3riAdmTOn53klAsw9Zu", "proof_sha256": "d4084404956eaed0e24b8d73e0fc9b3d0c8f8dedae8bba40c0d320b8ab5e641a" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793114634-SRkc2KMhWHxIHh85S9PX", "proof_sha256": "b8614d50718d32e75e24b4f590a87aa2343c1cd45d5ad7de4358558c343e44e5" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793114673-sudNlbE0Vk7RR3Zu0haz", "proof_sha256": "1c9af0b8363fc8945bcd072103056a1616efdff59d9c2b7bab2efd5aaa254e1c" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793114710-WrcQAUshoXZSgztkn7Rd", "proof_sha256": "54226a3036272a9a134639f5cd2aa5442719b1ee1a928bfc8e6b5c3e960bce4e" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793114751-RVIF9bSVAtXiUywg3PDq", "proof_sha256": "3a365e78331b442e90a1740c7e0fbe2f42e4be3efaa26b90ba693cc945820eb2" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "the scenario involves a business owner suing an insurance company in federal court for policy proceeds after a fire destroyed their warehouse. the insurer denied coverage, alleging arson by the owner. at trial, the owner attempted to introduce character evidence (testimony about their good character) to prove they did not commit arson. the trial court excluded this evidence under fed. r", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793114789-N5FIyywYjTpaxDeZ6Dbq", "proof_sha256": "60c905493ed39cb065580845f9577c7a218e76df54b499603d6757991f185a86" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793114820-fv50DyeEjzzlB78ySGuy", "proof_sha256": "45216502d0dfd4bbd8df0ba38e471d668372f44ec9cea5f295919d02f5ed581c" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793114851-vmMoLsvLOGce3yPVThfo", "proof_sha256": "149b5509af498c4a9d3c1bb54469a928e2ff7d95576479d2ca615aa64b181b1d" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793114884-454zHRaUDW0L0slBBozJ", "proof_sha256": "81b20b476a6bdc7a874a011e720a2bf9117c4f350fbcfdf7a8e075cb48da596b" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793114906-m7uWKaQ7dqX3pu5aKn4T", "proof_sha256": "e6881c22fa6f225bc924f5b7b97fcd9fb98b81520d68d300705c1da499ff61e5" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793114924-MgRumd7RJNiO4m5yAJYP", "proof_sha256": "11b9bc96ece7481a924e9e638532d058b6d352be6986d66593ee31d83743bdfb" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793114938-xk5xr0I44y9DGzJFHTZ3", "proof_sha256": "e267666b912694903e9973734b77a48657ac2d560d7771d221f0b4f67a38b332" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793114959-ZADgs89osvjG8QspbtM3", "proof_sha256": "09c12362588e088fad6b9afde762945e3d9602f9ed9d4ced6301762b8960261c" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793114985-IGtWWGeUmVeC8zOBNALo", "proof_sha256": "8350e04b623b244b3dea04a852e46106420836709fc8a2a2dbb2fb741061106b" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793114999-nYAut6SP9LoN9Cf2JPIp", "proof_sha256": "8b203b530e479468fc598160864ae719265dc30bdb1c3e4e5ca500b1d11dbacc" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793115011-eGbOcMB83D0g3ueyZ7KE", "proof_sha256": "fb2cc389028814a84446ab013465a720ffd3910f15716fdc2ddcce4667fbcbd0" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793115051-WLUlCu0SALrQ6TcHpD1E", "proof_sha256": "b3efd6080188aa2a898c9bbe53563675f01946b5fb4ebc37e3633dc81c9ecc27" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793115064-bJTiqNTVsHYIy6Z43Db9", "proof_sha256": "80c5cf05808511f00fed47dd2bbc97dc871ed247559da6ed6e28d97a7464d81d" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793115075-WoloaT4Q0wcsUXFqBs2U", "proof_sha256": "fc91845352ec641258d9ec7ea78564469bbe998820b5c69847011d7cd1cb45fe" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793115094-lz5OSYLjSdIP5TCwOTjX", "proof_sha256": "f03db653e9ae9e78582f718c5f08432be12a47e49c1e2c3938370acf02f0c9d9" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793115127-9zFNtedIxOrrPEQ7Q3Lk", "proof_sha256": "b70f8aa0ee8bc9b20d80cdcead00df7273165fbc82496839ac7324161f74eea6" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793115161-l6hu5eiNgMaOVqhrnNe4", "proof_sha256": "b96edfc9e73cc0051f289f06aac6a557ff4086a7e682ce59b2399a97df0255ef" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793115172-rDQWpxONG1g6sD9TpOB1", "proof_sha256": "d1afd2478baf664665fa81ed104440302da89b5c4fa7939c27dc029625328694" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793115193-DkBiqqoSYFVXFxVJWejG", "proof_sha256": "4daeaeda6866aa0461cde0ab1fd19c6571f13fe1e0a34b345dd4f003a8b443c1" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793115214-bMFHfbLtVi4p7nAOzTnQ", "proof_sha256": "4a7996fc441dba5af7fd912b00bca885a460df6d62191cd3b4fb5fb3e3e2f2aa" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793115250-VhZfTWmBSukmXjZi5MKI", "proof_sha256": "5eb1fa676f351f194aca04af4ef8d4ffefe41ace672f522c85434f6bc7d8861d" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793115293-isrMIDEsdyqJ8yX8aLic", "proof_sha256": "4749f1a8b94cc94a56f6f61255df1c3bbd371902630e65bdb19808397e059897" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793115320-qF0RSynCr9nhH5jyyAnZ", "proof_sha256": "ec30e02d5b9c37bd1877b5211a48720e83ccde1465ab79e0d56586089677cae0" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793115358-3nLUcUaiIPSoOnIlBPll", "proof_sha256": "4adeed06ab13f90fbb58112115f446780a18f42c6d7bc03749a447fbc59f6511" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793115395-6PqFRmQ3BvHB9S4bedxv", "proof_sha256": "8cb062bca0b916f7d6b7d4201786034fe118ab070832f0844c3ccd0ad8067ca7" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793115431-Sw9WO4Fw9cw3tMf429mL", "proof_sha256": "05efac671423e674628b63fa305f957e28db601695cdcf8c071dbd716f224543" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793115461-91erJ37tVvXjRpisjTrs", "proof_sha256": "a609724cebbe336c9b939490c3e819c8ae6f780a5d99194ead98d5918754d395" }, { "model": "mistralai/mistral-medium-3-5", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261027T120517Z", "generation_id": "gen-1793115489-ri9EfCrTgZsm74H4a45V", "proof_sha256": "d48b0c5ea07672389fe05ea5eb40970a3ee6e7bd81aaf77601344c2c44cfbbdc" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793102717-wvfzeO1rpG98N1OTgzDg", "proof_sha256": "c65693662f039124324eeda7afcd1a83b55e619b4760926b8a6c097291652ed1" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793102765-GCe8WwYZcDw8OKtDdYG3", "proof_sha256": "2af2918d28b2c327d7bb8229b2c2f4ae5f1d3f882c2f3a1993edb3e7c7b10cf7" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793102805-0ct18lEgCsUJFZo1lKgi", "proof_sha256": "82bdd9d0eba3b318e71ba63fcb604cd1839e3052f963204cdd48786f363c179c" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793102851-jegEWmv03nJZkmjhSE76", "proof_sha256": "c983fe5e188c866cee871ca63b731177a5f99c0d9f1e26277146d7c63e0fca5c" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793102911-u4F1D73AMN8kDmZR4mcD", "proof_sha256": "b1553ff3a0108d5e08f0f278d848976dcbc34fd28a45f88be80e00b5982be501" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793102971-SXxc1JB5ncw2K4HoYeZO", "proof_sha256": "a893cb50b69acea6e6ff33c39b9ae55669a4ce549df735be78f4a72d79143b31" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793102997-BJHkoZ7Ndxxz98PKNgt9", "proof_sha256": "88c4d8b370ff8dd5a93667acc50408e169125f0df8d2f93cb1d289a328817348" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793103051-FXzC0usbgP3UzriMmScN", "proof_sha256": "87936e4c8ed027f9b9ed65852990e6d3aed8de530dc246ce94ac052ade107aeb" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793103108-YyDWbapJHm6VIVvj0KjD", "proof_sha256": "831422c423b4218433a8f20fe822dfc8210af251724496570f4b9ff0302178aa" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793103157-7iAljrstZf9q2BHJ727b", "proof_sha256": "6af8eeedd5e86933dfdf71364a5937c0c2d6777a21487714fb39b5bc831166cf" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793103217-3jT4XlKbQ8aqnxkrNPYd", "proof_sha256": "f6154fd8b7c34cecedfb83a5d1c4b4fc801486b1978157b1cf67bc977c6bc448" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793103267-P8Wa9mnTgnL53mC0wxjy", "proof_sha256": "07d2d2f07ff5d14bf13667ae8200c4f3deaacce12c37e220a5ba2a0b54851854" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793103320-6bQ7sPsUnTTZaX7nvzGn", "proof_sha256": "314eb88ede304736d306a1b5761e059e089bff89d5b18e52eeb62904dd23c1ab" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103352-jpbTHCDJzILdOwXLVd2t", "proof_sha256": "81e0da8b7fd43f424d1c15777e4769762374d15db43c7eac11f39a0f4eff7d04" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103395-ZeogU7NAupRWgEWG2Y7d", "proof_sha256": "90aa538d3afd3a768609de720fcc92d19d93ce88d5fa74ef0a7d4318c6954480" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793103432-3TgxrkzjOvFch2uUUlvV", "proof_sha256": "d61176e8911d4b9d87a0f5e96550d1f53ef2db93b11082c17235a144f12407a5" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103452-O2RnnQBuXLpwinzBOOPQ", "proof_sha256": "ed4d8ed7725ecb0cfcecbd074eaf06414e168df9ad2f43380d0311d71fd7c267" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793103486-quWVnsW6qjeBw6wPSMBv", "proof_sha256": "620cdd47aba6da3c0f6ac39b9109bf8aabf76c5f78dd75ad68dad80479097f3f" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793103515-L44KoXCGTIr3T3BXoP3A", "proof_sha256": "03b38a66b382619632f4f1e6453a6b7223704e60768305bc2502558ec12b629e" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793103528-Bg7jSCbt5zg8Md3ztdzn", "proof_sha256": "c29ae618ccf3ddfb0adc2b64461068f4de0dbba57059b4b3398e7b8733adbd30" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793103539-igNvnmKwlgHpH0oPdWKQ", "proof_sha256": "b37cf19649fe8b149524b6afa69ae30569d980d609d7d2a377b02902062b37e0" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793103551-9RScWzX3lfyVUbnix8eD", "proof_sha256": "3c520954cfe80132bb66986991ea9b484af215cae399d482d38424a649245c9d" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793103568-xsf7bCLzQ71fbFp2HkO9", "proof_sha256": "303fbbdc2fcf2728cde3a41649c4f9d05ef67ed78929772748221ba403ac6e42" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793103687-udatQq5WRH5a5ZqyvPmc", "proof_sha256": "3356a717916b00c67b11c14768d13f5560aad429dfc34b6e617f87d261606d4a" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793103716-iLcZMV6qeiYIv80tEWaj", "proof_sha256": "3be57311d01d923b342bdc1e598d5120342f242d3c3fb23c3e00042effe2d419" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793103751-oimu6rmV1gc91VIhEh4b", "proof_sha256": "51a4adb00cc7b9b3242b31371555feba26004dabf01381a3f6f1b4d6f08a0b8a" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793103789-TFgbZjuQcE58PZIdqd1K", "proof_sha256": "9f2176b7977adf26237e40b664313a54d1d0f39c35e0965f399fc30ab1a42f39" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793103824-4JD6Q0l4orSbitKrzqKB", "proof_sha256": "0cad5a3596d3152833128377e5cb5d76e51b58acf7d5ae97b12aa0803f9570d3" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": true, "finish_reason": "length", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793122291-Xk0P88z8sxRtLPv0BBbi", "proof_sha256": "8dfee74eea7a48362056f3697903d1afb49b4c3fe4caecc722530e5c6c4977df" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793113930-k4ZgUsI3Q9loVuLItQQm", "proof_sha256": "b951c37a8f55907feed42c227197c3506dfc348a2afad18a4f7634776695fa98" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793113964-o409AYLN4kFWD0YZKUJ7", "proof_sha256": "1ed102463d8b5d3ad9091b488f708d39efcd5e04888de2707a5304b31b9ef3a7" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793113995-lB0VXaAF1aU0iFNJCb3S", "proof_sha256": "fb7141cbc40fc954148783aeb537b03fee012b962c612335db779c6cca77996e" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793114040-pLmzh3n5XSOFk40qVWXg", "proof_sha256": "e177062f9f0c4d01b5ece1a8233a8e2c04818568c1fed2b308aa2990f3b93237" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793114073-rtolbbSRxouFM75jheav", "proof_sha256": "59e7e1c5b17588b0d628b57a3cbe425cb849e0b6794f45389bc273812535d07e" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793114105-D0eC75abQDfvZx8un4OA", "proof_sha256": "39d5c168006cd16a8dd6b763be1847a2b51aa7a64116347421f0a5aaefa1e42b" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793114142-nGDtALz4PLa2jFqRITXB", "proof_sha256": "6c0b18f62d34ba0bcda297e32ab88e1077184de333a11b95e1c360afffa59b4e" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "SiliconFlow", "run_id": "20261027T120517Z", "generation_id": "gen-1793114159-0w3xNpQcUaRe2PhDzpTS", "proof_sha256": "90cad04a62b9a2e7e33bac00bd91fdb72d00a0459b3c2f3f43b1c74c339b6d42" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793114202-S9XCDTivF0nTrI0Nrzg2", "proof_sha256": "307a833ac46ead4a0ab1a560e628a8871c767b439fc33b625d552f8a08f9ca7f" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793114233-aHLmrtmWqaK98zz2WVRe", "proof_sha256": "91a5aa19c091db4736ae4f12ca5f7f25062f4f19c7fa2e551fb797a4daad9027" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793114270-8rP9xb0xDl8SgwLDPPX6", "proof_sha256": "502dbfd6824c4c70f4bbff5c080829ad9e560cab7e3c8704cda0173847404f4f" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793114304-b3TspW0rEnc0Xzskm55l", "proof_sha256": "7ec7bf157f849b4256c3a385ae39744a297eff1f606eb845d9c86b3a9164e6f4" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793114346-z1F0kml4NIKKU7io0REO", "proof_sha256": "f5110aca3984deda19e75a2059032bd1220c0b95017337e8c3800c1428f5f3cb" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793114386-YAEUgZbmOAbs8XMd1Y3O", "proof_sha256": "df43338598a1c9f7aaa7658076c2ab1d8972a2422c3fd61934cc12f871af2529" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793114425-VZ5zYllkCb6Q78Hxo8MJ", "proof_sha256": "3673a18fe7dca73e7302fb957706ce9b9327ffbf4b6e96f880b5a7197ad1dfa4" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793114463-C2JaNcf9LlfrimqQJJQf", "proof_sha256": "e46c7b90d59c8ec586e26c0dd365939867d3e2901917c23f5eade41d5ef9ba60" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793114488-dWNdKQ6l2YqUNF6CTQ38", "proof_sha256": "04af86494772ffb4999e7d143cfac938a12a0b1b3051570f8ec4742d30aa418b" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793114514-NkuGBlhrjCEZOFP1DhgB", "proof_sha256": "5d3ee794be52824f80e6503a1813bccfa84af551070dc81c03c7560ab17c8fa9" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "SiliconFlow", "run_id": "20261027T120517Z", "generation_id": "gen-1793114569-2jzKeT3XgWxds4x6Zfz8", "proof_sha256": "ae9ff705c8f125a73481757840438df1d3f445266a0924b55d2ffd8b10e0cfe5" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793114657-e0fgHjW9IYm8gQGcagb9", "proof_sha256": "8d77903f51869b10ab8f1e9b1ffe59870277e8877290f13d38f39ed3c5e5453a" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793114700-2m0U8yTVhFvVkPJvTObQ", "proof_sha256": "62047c3c0c9d95d68fb63df3a94634dfb8fc78d6c4e42a621b4ef7f355c44508" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793114723-u8qgCtYjLiXN7WNWEkAj", "proof_sha256": "3fe1e15315acff2c34fb5363204aa7cf82e0087f519e0316591fdadb394de507" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793114752-0F3Nl5sSwHeyA1LVajxv", "proof_sha256": "4d28d7491ff0170bdb39f79f9d46e32d80e474b634c1b0716be9228194e2ef65" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793114796-P1HvD2c7T7c7v1kTy60v", "proof_sha256": "10e030795b7a70505e970764db88b60bf5b08eb31b08c03d839098e49886b328" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793114831-HaP0HaL61qyTWEaqEYqr", "proof_sha256": "7c170e7b9dea34dc3ebd85e3961bea8158b6a736f7bb2f5356b8f02b02763fdf" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793114865-YUeN581SbJipRCb4ySnn", "proof_sha256": "10493b61db6bdf474a3c87097138cf3b1e04721c31c0a3c64b65fdd3aa578747" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793114909-hopavZzd0u7EWCNrINQY", "proof_sha256": "a7e9ea6d19161c8571cdf391b95c2e8ba8a273223b25b3ccd9789adf46470f37" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793114939-KgL6ZjxaVFBcED4cAEU6", "proof_sha256": "f918420eec4722463bc2b4cf0f66ebad76b764ecc4babfd562750d5b1e4894dc" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793114959-2Pc6vPDHwmqaCR3SOGR3", "proof_sha256": "dda3cdb7bf34758c0fc9fecfb6644d66e0ec9e227a71fd148609384a376e0a9f" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793114978-pUEAMZw8vCw2K6VatvcO", "proof_sha256": "1aaad4ff204ecc68ffd2226d8e251df9631884dff1e8607a124acce7aefcbd46" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793114988-iLcnEwKR4E6Fn4nGfKCS", "proof_sha256": "85d8a0969331c64df32ef518c7d2896c871c9f0f5a0f139f986a9c79de718aa4" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793115011-3ruZ84NidlsY8ZI4kRcq", "proof_sha256": "1c182ebd2691f5042da8de3dfe3b68ec9fc488899383245ad234df841cd00c4b" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793115020-pPKJFMt2WHHXRSHrWxZB", "proof_sha256": "4901c6eb8461043c639bd12dc8b26b5e10c981ad44a4ba712816883da542757b" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "SiliconFlow", "run_id": "20261027T120517Z", "generation_id": "gen-1793115028-RDoullY11D0zXREm1AS6", "proof_sha256": "13972a02ddc9f6e9585f6151d74b950a69de03d33e7e346225a4bb1242045535" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793115054-UpDn8tdC5IdNPF1Hqx3r", "proof_sha256": "48e297ef2e1e9d548726ea17de41ecd14b871abc8363a0b071929790addb34d2" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793115094-pC7A1eYO0P3Fc8SeTYxH", "proof_sha256": "5d485b6b8354759948170aae4ccec91625f2bc23557e05859c23c6a86cdcc21f" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793115127-AdWKSU6X1PZBfmfG9BZN", "proof_sha256": "5fa9e23cf7b6e3060576c939e7837146984f9e202b32b535145acd8db7dedbf6" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793115167-950qygvDhlvM9ayJkI1Y", "proof_sha256": "add9600d0c82718a2b8d0b0744c7cf19552b8313efb4b1056f205f4b8f49b524" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793115184-zTDkBahcfE7u98LDItFA", "proof_sha256": "e00953871d3392b504a2008c54a74dd5ac9c495532ce28f312a1a85afba52256" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "SiliconFlow", "run_id": "20261027T120517Z", "generation_id": "gen-1793115218-s2A4RWJJg5528KUW3bkQ", "proof_sha256": "5e17c3483136325c52aa1fc52b92f37573666a634c87deb1a92da28f1128b712" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793115248-OedXFBOJfln4S1aC6DeX", "proof_sha256": "460f73ca2f08693874c9712c48c6e8c7ae5a8dfa55537fa38d3774c81c1aba08" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793115266-CtTJGsFtKcfF7GMzIovQ", "proof_sha256": "b94245aa410f671e6dc7cbe96a78926d21a8c99c2b5137a9f4d77cde767a9d43" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793115302-nRMk5IAv2qlcwhNfv6Vk", "proof_sha256": "b6cc29d808f976f6a94227d4e71f84cfa14caa7f766046449a506c5474473f0f" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793115329-Pi62fjNL0rdER2TyZVUn", "proof_sha256": "7b084855c31a0dcc623892af31325354c806a1cc78837691cd809f8cfcb2161d" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793115347-xwjR41p0aBic8meisL9h", "proof_sha256": "69198f082b46f321f7d304f1917c21361bc825151c253f688253a8568f9b463b" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793115374-gUU61A8wfSDG20miKjo0", "proof_sha256": "fcb822f79ac255ce20315011583886f85d0d92664967a5f1e56677384ad6a9a9" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793115387-MB6yGDNMqTTwoZbzTkGD", "proof_sha256": "38b7c73445d4b772d18f687f7bd68941340e15cbe59061d06882ef6c11b5bc53" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793115452-0rT9NDwoNNnIWnLVXin1", "proof_sha256": "6a427543139582894f93a57abae2f6edb32db68fdf0efeac9feec94358d748b8" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793115493-vQnUmdYy1m8DIc5eF019", "proof_sha256": "95398fe961093648cc285770fb2e82d49ea53eede9e7d734a194bf06fa083c48" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793115503-nsHY2SvlH8fSO8J9nLl9", "proof_sha256": "9f2583537ac5632a6974d7c81b30755a6eb6871da588d998f46f52e17094169b" }, { "model": "qwen/qwen3.8-2.4t-a95b", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793115512-QbhJ3iCamDeza95QGIfx", "proof_sha256": "abe1cfb66252906ed5e4dc44533f24ee9ddf9521ccb0fa659164e5843fc84c87" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "SiliconFlow", "run_id": "20261027T120517Z", "generation_id": "gen-1793102717-WMAMcJvBdrzAydQAYGmP", "proof_sha256": "c53b56fcfcbcd32b513dbabd6e9790e11c15302ceaca61c84f2787f940c16196" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793102752-Os4ZMVv7NlXyPkDLBCnq", "proof_sha256": "685afba18a851c574d9da1787e6885d67d2f08d4de766d40a6ec75d7167ae37a" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "SiliconFlow", "run_id": "20261027T120517Z", "generation_id": "gen-1793102799-FGG4rdcfcFAk5cwRO8kx", "proof_sha256": "589f3ca05ed0bfac0f97b5508901b52cf36aeac06a6fd95f5135f18331b2eabd" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "StreamLake", "run_id": "20261027T120517Z", "generation_id": "gen-1793102885-zyt6IlmhdfCULJeY7rNp", "proof_sha256": "0ffa8048410aa3287d330b4e3c904f84e6e668eef7e2503738eac1ca8bcd60a4" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793102988-IAWIRSRnpi1sUJcwgDl9", "proof_sha256": "4ce93daae7a8dde76bfc365c6e083228798d5ade9b08b91b570024516c5851fd" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793103151-XBSTklTsf6JcDbf7LfuS", "proof_sha256": "31f812086e625aa1d6eab35dfb5c677e711450770e8498f51ab53072eff00ef2" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "StreamLake", "run_id": "20261027T120517Z", "generation_id": "gen-1793103174-8TnqFCwEbKeNSBlN6t4Y", "proof_sha256": "d8a355877b097974ea7299c39d37427095c9c30feb084b0c62e1770f48936a1e" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "StreamLake", "run_id": "20261027T120517Z", "generation_id": "gen-1793103231-p4V9PHr1IJL5Pi8fdsKk", "proof_sha256": "9a9fa3171fec904652ea664058f9ddf386fa1631ad39ad40ad8db9968fef17ca" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "StreamLake", "run_id": "20261027T120517Z", "generation_id": "gen-1793103345-uxmGIp4qwMISgRsjw8f2", "proof_sha256": "d363cabbbdb3dddabbcdf303afc212a91265f45710c245ad674807c32e1ea170" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "StreamLake", "run_id": "20261027T120517Z", "generation_id": "gen-1793103423-MWhxZGU4eFYHZVPXmGDw", "proof_sha256": "028f68f61578b70372b154ad0c57d5ad4c543b947e3643466961b6cb7472e89c" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793103522-ibXCgeUotwfFh80pgex2", "proof_sha256": "c9d943acede6f0e6abd658ec36aba746518f66f833e40ebf355bd67b7b5f87e1" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "StreamLake", "run_id": "20261027T120517Z", "generation_id": "gen-1793103534-S3Q7qeeZn297WzycjJc1", "proof_sha256": "0628aabcce776ba286ddb2052117a3080809745e90eb73a4422d74533ca3f7e3" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "StreamLake", "run_id": "20261027T120517Z", "generation_id": "gen-1793103619-qsEwpioIWYvIEFMuRaEi", "proof_sha256": "5e3b991f711ab6b81627650b927ef189345ea97d3b5e0f40474940ff750725ff" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "StreamLake", "run_id": "20261027T120517Z", "generation_id": "gen-1793103645-pOgJOFNjAnHp1eTkOtff", "proof_sha256": "0d3f7aee8fa30cbe72d50bb300ebc53748d1590be1c68746173d800fd524c903" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "SiliconFlow", "run_id": "20261027T120517Z", "generation_id": "gen-1793103718-Rp1SmkMGmumrqg2y4Gat", "proof_sha256": "3e8c49bb45558c2cfc509c3818b041adb81b2d704f2ae3c1f9cea52882321a41" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793103761-qW7TktosOl4l3Vnati5P", "proof_sha256": "615ee10b7c158ddd1a75d059ca9b0a5cc02d81f1924e3fdcf4e3db8afdf42925" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "StreamLake", "run_id": "20261027T120517Z", "generation_id": "gen-1793103803-czkWqR5amTUVnGLbJBMs", "proof_sha256": "f070bce3e746a22f6752cf304f495783631bb7c22c45179d3275bb995d3201e6" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": true, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793122291-zBuOvm9pq4AWUPauJsLL", "proof_sha256": "2f0cdc3c555119c33ba553ff6acf881e4097fdc4ece77e17482c3198f558670a" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793113880-u2N4Qtj9WSm2JlDbVWYd", "proof_sha256": "6333478edc75c19a97f21963ef8d000f23b9addeb4d68a04e80a24fcec0bb16f" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793113895-QVd7gDjLSKPHzna2ZMLx", "proof_sha256": "fb52168b277ddddf0a833212bd731c977d8a4691e3801945a93c41d775dafac0" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793113906-TcG79xtUnvYR0Yt3oh6u", "proof_sha256": "70883f20af42122ab868d13eb2bc3205f067f2bbc49a2c4863717a03d864ddbc" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793113919-f4EqTBqwF8AS5gskKBhg", "proof_sha256": "ef3ba00a58d5eed2f2926ea51e20922544cc2ba14f0298e14d97162ac556fef1" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793113933-A4lhOsFOewWUFVqbvXpO", "proof_sha256": "f60cd730bfbbbe884fce5ebc4b3f4f6a208de1b61ca976a22f8b261209a78847" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793113962-Y8lR3rwFqPYZ8djjpzRR", "proof_sha256": "0feea7dcfa7d0dd5dd6dee6e7e0541e47d079f5d37160a5706f72a6547131bec" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793113989-ci3rv7e9BjHobwuiAiPa", "proof_sha256": "b9487350083bc64017c7cba221ee9cc4d1cf1c9c4a1bfa48c9ebe9e54bd02ffa" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "SiliconFlow", "run_id": "20261027T120517Z", "generation_id": "gen-1793114025-r5cD0WS3tnhQVeZf8nAX", "proof_sha256": "4337e336bcce81c512912c472b2f3f5062f3e326fcaf9de6f1d0dd5925f452ea" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793114080-uawKvqYmdTwmW144OpUl", "proof_sha256": "6c7107272da7668e2cb502bdc6af3b655e7c98437c9a412c487fd53e0a247e5f" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "SiliconFlow", "run_id": "20261027T120517Z", "generation_id": "gen-1793114104-kefXebTIEb42tjDiOLNK", "proof_sha256": "54739aeb6ecde4d224592e18b4f7a410b821356adc8eeb1f06fc9b0829a2e657" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793114166-urxuFcy9H4QCPsX7fKAb", "proof_sha256": "40994bb0f829a10c11b0766423d2cf91d0a9f0df101d57c6e2c27c55f7fe869f" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793114215-NOfsAH2aB3l1wrg21exY", "proof_sha256": "3c2d88ba1a666809f0ecc831665583253dbf1ee7f6fdf28b31e75e9b91f15be9" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793114258-nlLO0sGiV0EH4jCa9VSR", "proof_sha256": "a2b5143ca9e85c0c29ad6416c868f0428479750fe1957c3382d323db60de30ef" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "SiliconFlow", "run_id": "20261027T120517Z", "generation_id": "gen-1793114336-5tcjrSlW27ZHzRJCJwPf", "proof_sha256": "7eea90e64930b91dfda84eb40ce669ed4c313aaaed873072234aad19ab486aad" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793114384-kbg5epjWdBlKfCgMmsWb", "proof_sha256": "69748b7190837b9edcff41032afc7552583ebcb8b090e478b4cc12958329f329" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793114426-F7sghwtTAedws1lQm5Ln", "proof_sha256": "06e45fa808406f11a5b2eb92808e1f12f3a43300fe86cec94c5b4448b51d6dc9" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "StreamLake", "run_id": "20261027T120517Z", "generation_id": "gen-1793114462-1PqpBVpA3ozumNGMstGC", "proof_sha256": "98b3f57256f5b7c41637628ab73d60a127fd0204f2f3a0f001b594eb62ce3b9e" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793114523-Ot340xgJiGXFjqJy36oH", "proof_sha256": "89341279faa6a4116fd3d26d48d6a2b521c95816a59b004f013b8873e816611f" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "StreamLake", "run_id": "20261027T120517Z", "generation_id": "gen-1793114559-24AiG3KHwKeh07F0rc7Z", "proof_sha256": "1bc61dfab7aac986a5b7ebd4b33a6c46c82a0112a6303563ef40fd5cb05f1b16" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793114582-p9gnZd0V0FGk9M1srFLp", "proof_sha256": "c7d37830c9126ed2da004110626ae60c24c024baa98cf637e6414f33d59e076e" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793114627-Rme7QaH8HO8EKxsAQRXt", "proof_sha256": "83e8ba4e11d47d89ab3cf076342df97a651fb3ff4b3e38e491edcb7860884bff" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793114704-wFInFCiO14nNXwnNHjZt", "proof_sha256": "04754f6a279f0bc5e5dc0c97e0b3ec778bce270f8cf49f196b7306b400f0cf93" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793114745-A0F3kg2kQwpwcZh1vnOY", "proof_sha256": "623e3846327497c9b40f96bc8ab800068b9c770bb2c73adea69da2e0dae4c4fe" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793114789-HLwfUCQmuWNZ2sGkOcYn", "proof_sha256": "c3dca1798c4a35ac5864335a3b8d848258340eb4b1478e9bbf2531d7c7b44be7" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793114814-qr0pJ7J0V7BJhs4Ewn0e", "proof_sha256": "5365bba7efe79c085422c3e834113c701393cc629a0e4c829b3892fc9e87d1ab" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793114905-w50lqO8AReedZctRnqcL", "proof_sha256": "3bfdde2416f2e9a18fd9702c16bb9df3c04f9dcd7d29190e9ec43ead6c84e263" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793114959-cBuoAJDwqDjCZdExGtIw", "proof_sha256": "50a00e2662fdb3afcd1aca1d275d1a5801bdee1af0609e74d6b780f8dc1fb1f1" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793114979-zUU3wjpw8tJRw0AhvO9q", "proof_sha256": "620d35c2e666bdd8654bb1192d2708c26698a6a757be27bba3d604dc3d8e2b52" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793114999-cdgjZjV4fYoLUuXzzJy9", "proof_sha256": "ec6ca9d88912a5045b894a8e34afa2069f47ef88bec86d12354425f57a9b4ed6" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793115010-b41jLsmoKuXcog5aHi7Z", "proof_sha256": "aa7d7acaebbbd645bf925f3a0395d3f85ce2b300d6a8346e56ed84575fef3a1e" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793115157-c8yjyXBj7MvVEozSeBVX", "proof_sha256": "00443f789c6371eb62cbff23f5d9c46e151d9044b9cb3313e8f7049bb5ac9464" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793115221-gDBkY3SDJfYWvz60W7zE", "proof_sha256": "753fe890f91c4e25731bb237ba505a81225a1cd200255f0d26feda1bac3662bf" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "StreamLake", "run_id": "20261027T120517Z", "generation_id": "gen-1793115247-wPDbERiu2JoCViM0l6fV", "proof_sha256": "106f899b2682211e069ebd54c0a94df4e2f0e0fd6bc7b40ce33e2c2c3761456a" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793115300-B3kqYK9JItSDllRdbX9g", "proof_sha256": "ccbb6e19a6df3ba28c4139de95f5cdb69bc9a0635fc2f65d3de29182fc0eda3d" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793115344-Td9E97FDGV9oTgIUnXcN", "proof_sha256": "895e00c703e43b9b34ef9b53822fe953fbf4ea118dc09480b348adbaccfa4e93" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793115371-ZwpB82Fd26pMvYtqjg6l", "proof_sha256": "b931ad6279fd078688cbae0e4327d701446d6d94aee26270ba63d7c727ed5d9b" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793115417-5Xs54FZ9pa7Q4I3JOIk3", "proof_sha256": "40a42f11d7d5a5b5b367556e793cdf622cc3bb4594dfe451b77269dcf949a153" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793115488-WBgPTzI4PT9HBog3LOnj", "proof_sha256": "dfa8958cf69799b0cbac777a53b0a5577cc534dd2ee698d3631dcb61f7cfd366" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793115514-ivQ5jTwSXuZ8tHSEScc3", "proof_sha256": "a42f2bf29c33eaf8d379cd9cea1d856118f012f74cff1ddaac836e3221ce157a" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793115550-cNwQHSbQY6jBwvHCDSHJ", "proof_sha256": "c673cfa3584a7bc9d4b3d233e9573411c070f827095ebc09b02b805b9a04b7e0" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793115559-Q0smI5dYxiG8Ht2FezMl", "proof_sha256": "7c9e077048acad1a9d6e35e50490e12379ab9fc196cd1f6b39021d24b5fe4963" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793115568-dN3788dHq147clONleqb", "proof_sha256": "93e449031161e8585b8a91c2929842ee8acb4ed9d2f5d8888a7ee809ffc23e4d" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793115577-UZuXWeowTZqz82hxMZ0l", "proof_sha256": "ed83d44ed08073c0ea8fc44ddbad084c9da071959b3d88c3dcc831550c9a2bd4" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793115604-JBaIsR7L45Ol7entmwcl", "proof_sha256": "0150b7c5b3b8f595501ea59b84b4e00ab5d2900766f131afe26eaba2ab3e66b8" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793115643-M5Myk6lM1jTUVt301xPN", "proof_sha256": "4840766410843623694c43c422cb75a50e6a0811e324838e62ec7bda39bd191d" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793115651-Qch0elkdpIUUGAy65y9X", "proof_sha256": "340d3401744176d0912afad8882d7bf319a2a1a56d9bccbb86a3c8fa538b6e96" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793115661-e0h43qRhDNFQN36CMmSm", "proof_sha256": "d2b2c47ccc4904d97318a6eb6b674d3bdd34567da18b421ff5b256389490777b" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793115701-KTaYdyMSdAuYoHOp7BaY", "proof_sha256": "ae566083cef5e74d7fe3d908b298afe38a85e8ba6b3721584422dc01baa3d57f" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793115719-Qg8h65VWfAOCggW6Ehjc", "proof_sha256": "a5ab47ee09c3e6db5a4c7faaf40245a6e75822fdd044935e298c46593735d3af" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793115737-MmV18hkuTiKIhXVQkYac", "proof_sha256": "7afba210479117be8b839dbfb7f9c205ef403366582bfaec8d982634d1c8a7d0" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793115759-ZKaRFv2CK7EBj5JFOnGc", "proof_sha256": "790e30ef2531a7657805a46374b41cd350321ef49ea1ded64612ba49e5306f0d" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "SiliconFlow", "run_id": "20261027T120517Z", "generation_id": "gen-1793115814-jYPH1dP9g7OCgC30wy68", "proof_sha256": "abecbdec472f84e926806162a083ebf61eb9c73241f2f3d642711a0bb8df4bdd" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793115859-ftbu5QltbbabkHc6Gvyn", "proof_sha256": "79785c34a3d78cc807d69b95b7fb531104ea94b37d18a5d2568e9d9b2417fbaa" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793115899-zeK9bvziXWX3xaqX5HDT", "proof_sha256": "c3b70034c715ec18e1451b020e5c398622c1f82b1d2b58ee85d3c60f0f0e36ba" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793115958-jSQwiheVvPBcvr2eHAgz", "proof_sha256": "21972988fcfe0fc285bf78780b72b5454b3a3518ca71418d5050fa155b04e514" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793115967-MvCApobEUXsaZkHbJJuG", "proof_sha256": "406bf5fc974f5e6cb241b4be87ef7d7878ac907d6c3244894de79a23f9534841" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793116014-jjHsDnlqzQBji4VGakv1", "proof_sha256": "48a46bed1a1b32a7b63a4f4aae26ecbf594f99d0e9d24eddfaa59327b628d673" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793116035-uy5fTLw6minWy5WZp55w", "proof_sha256": "d891c7d7f3421e008b104db13704598c28c9261182eefabfaea1d0ff3fcc4aa8" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "StreamLake", "run_id": "20261027T120517Z", "generation_id": "gen-1793116052-GdtXkZggyNHrYUuscqU0", "proof_sha256": "b25cbe1cc405370a312a5af7be02fdba11cba5dd2f7e7a60d2edfd925e83e29d" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793116069-ZRVqPhKFxxlEYK7ofKdR", "proof_sha256": "cda06f82b39925b0bc9456723692bd5f1ecc9e19a0673b5de4ce4c600720f4b1" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793116083-FNx46WXNS6Gywj05Vts6", "proof_sha256": "4b8d0d550a51ee0f96f36b1d7f1857e5731af54a377ec758fe455c69b660dad0" }, { "model": "moonshotai/kimi-k2.7-code", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261027T120517Z", "generation_id": "gen-1793116091-jCtCQU2ABvoXaCy8MtrX", "proof_sha256": "e4acb8c5f3b338580279fe72e6ad30da2b437d5756405eba8076b7bd460265cf" }, { "model": "cohere/command-a", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793102717-E2WX7SHWjpEw4GNyTHyE", "proof_sha256": "38f531c5529aa59013ea2e975e7bc6065e75a3988b8fe42c19601a7d876daff6" }, { "model": "cohere/command-a", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793102757-Z1V7Jb2pKJW4mNAoneUh", "proof_sha256": "8ab8646dae65ad7490de8336931482da74274e113b5478880ebe21cba1cd282d" }, { "model": "cohere/command-a", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793102847-hxEPuYqq6u7b0KUN84J1", "proof_sha256": "0729adbe6ed1211fadfa4553d92d4dc703e4076d69ffddd278c657f5ceffb2d3" }, { "model": "cohere/command-a", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793102925-pT4wKcs81FjkSv2sYoVl", "proof_sha256": "ab929b7672f08c06f95d1c4e87eee1a604b0b931df79d99b71c4eaa38c3a5e17" }, { "model": "cohere/command-a", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793102969-Dhf88gjL0z5YHP3w9KMw", "proof_sha256": "f894e022883571e500400490a8b7b3321251efaf95466fc7fadfe72c7e8908de" }, { "model": "cohere/command-a", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793102997-bBj9wXbWrKAM4pM2ZPSx", "proof_sha256": "0a4a33577b5f307ff00553949fa6b5a5244a1fe2b414836ab061aea184913284" }, { "model": "cohere/command-a", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793103060-9BzDIZd6v2EqVWn2p3GH", "proof_sha256": "c59a98ffc646e4745d825c392f9498602f9def626c8eed599eae1c3deaf3d389" }, { "model": "cohere/command-a", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793103139-7raB0yoDm3CvKkw7dzzo", "proof_sha256": "43038f9153037a340bd2f28429e3b55fca514ecc6bb89219a5f812ce3fe6c054" }, { "model": "cohere/command-a", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793103194-qAwMuQMUav8wdVKnZSyD", "proof_sha256": "3093536062fceeb1a4f824de3f415109a75210329b3bdb604ca3419855ab25fd" }, { "model": "cohere/command-a", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793103265-83ti9nZ12h7EXChS9ijS", "proof_sha256": "81c85617274fc3010fa3c5ce1d651e73f5b82e8e04ef6bb70046f39beec85e5b" }, { "model": "cohere/command-a", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793103302-ApywFlWOiOyrVBGcuD4k", "proof_sha256": "8d29d62eef0356f4cf248aefe00cadf840b752caff230256ec7f7ba983c80b2e" }, { "model": "cohere/command-a", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793103344-wIXMJxkXJ1YnS1KBlCoF", "proof_sha256": "809ced7c44888c5fdbd3fc9e5975a2a1392ffc580d78cc6dd5df974fd57d5afc" }, { "model": "cohere/command-a", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793103384-U93KBmCkhMVE22klRodk", "proof_sha256": "8f86d052cb18d618429ebe7f71822c15b72e942c6035e1c6a1e3d2c77cbb739e" }, { "model": "cohere/command-a", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793103411-XDsTKR5p7xK6QkniB25p", "proof_sha256": "7be19258c1e34969e38a8fb3b27ddb29deceed1717a9293d2418615372fa06b6" }, { "model": "cohere/command-a", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793103597-ENVFGACqrnrVL5hk5PGf", "proof_sha256": "e6ecca69219c272f7f54fe22988b2f5369878723e68bf0c3d1ca2c32568f06ba" }, { "model": "cohere/command-a", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793103616-1tZoHPxnF3CIQ87Kp2UT", "proof_sha256": "c4884d619c3e5ea37d6d2d377f0d30aae1424796fed1884156d7d1e687dd115b" }, { "model": "cohere/command-a", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793103641-Y2m6C9mQWn6f1qVSkfsb", "proof_sha256": "d321fa2fa0cdc0e84090762e61fc876bbeffa935fcb4239b9a21be9d959de55a" }, { "model": "cohere/command-a", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793103697-bA5zfCxI3fkP5dZlvpQD", "proof_sha256": "5ca02bcf68afd48a4c16f7db5c94f42707e105179b2e5b5a54326f5715d62fdc" }, { "model": "cohere/command-a", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": true, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793122291-oytQmczTjBZMpamK3rjR", "proof_sha256": "c15b73c942faf8aea5e600fb0a41aedbfdb26f97501a96b138159c582fa94289" }, { "model": "cohere/command-a", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793113930-uBI5hE0HQhVjrB9dPTkJ", "proof_sha256": "017b8a9dca47df6438990d3fef9caf962fcbb37b4da0e9fa2be3d51075307d18" }, { "model": "cohere/command-a", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793114053-g1fkNkdwuyVN4vd5bFWX", "proof_sha256": "a19ac29324fc3ae9e48a6130b33effd6bac724c82427fb715744f765898f6258" }, { "model": "cohere/command-a", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793114081-7oWCXlMny1GYrlxICUnS", "proof_sha256": "8b125058bc48a50319e16f95f7f35881826e346d5258b390e1ccd9d8ac2fd1aa" }, { "model": "cohere/command-a", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793114229-hhMStTJiViA5KpW8PZr1", "proof_sha256": "34a9cd6d3353d113fa3e591d2e8687d67410c7ac7bed8bb7491c883f735d29ca" }, { "model": "cohere/command-a", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793114252-fujNQrfkn1ttEzvOkj1S", "proof_sha256": "9daa6c3b5ff5678a3e4268dad47bf3543832938f6fa787ba21fc7b2b934710aa" }, { "model": "cohere/command-a", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793114358-i456FrQFWhCMmhOjN0Z0", "proof_sha256": "b80fd590cc548cc735b43650ab17f3cf3c2c160c7f298a67429ed7844729a138" }, { "model": "cohere/command-a", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793114391-SJo1j31Od4JdrilBYA5a", "proof_sha256": "b8cb873cc3c05c5eae1a271a4e6d24b867725f730f91d1975b70f30bf5b4830e" }, { "model": "cohere/command-a", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793114404-9yrA41wRFzPtU57nBBtu", "proof_sha256": "b8ba376dd8ba15dcd726ba01194e02d45ff3b20567486ddbff91f9b2240cd749" }, { "model": "cohere/command-a", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793114449-HZb7eL7ydoloNgrXv5yf", "proof_sha256": "c419edbd7baec1d9445dbc33ffa51cf3c05c7afe9e9a521294a2b5d6af96f943" }, { "model": "cohere/command-a", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793114487-fAMDwdIMA2rSw1OKNZ3E", "proof_sha256": "8603eb3e115cd97fac9f86c9389039310e583bee7208f0dce014fe02a33c4eec" }, { "model": "cohere/command-a", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793114536-AgNk2w6KqHyURL0XLOsW", "proof_sha256": "674e96223d8f79dcfa55f5ee2377033a657d4460080a3f20c94d7f6470dcbc5b" }, { "model": "cohere/command-a", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793114562-ZcnIk6uBFlXcg2F5IL5F", "proof_sha256": "5697362e2f13eda774f0cbe1a39322fede3e081412be5d6b95f75138fe95884e" }, { "model": "cohere/command-a", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793114597-Z0g6T8i91Kt2ul4SNBA6", "proof_sha256": "f9df6a62111baf677267beb18bf844edffbc3ab4570491e80fb7f64afb4993e5" }, { "model": "cohere/command-a", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793114670-clfVygCNIBrsp0PlQeCR", "proof_sha256": "b81d2d91e2aad4356f2e7ebe694ccdc855a8e362a6e7bae5645a6781af246c67" }, { "model": "cohere/command-a", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793114741-SRUrGtcBwNRdhOp8evYz", "proof_sha256": "fc5612c1544f0a72ee1ad368ecb09adfe1ec2a0287055291ff3a7324ab3c4e1f" }, { "model": "cohere/command-a", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793114777-dgeRL4mKhET7QzLJ6T3P", "proof_sha256": "c0d64fb39c62bca74202254bd2f0af3edb22eb9c9cb94957e06869cf4f5dc0d5" }, { "model": "cohere/command-a", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793114808-pFJQKCCsxOzc1znCd1sD", "proof_sha256": "41ba60b982607a40d49f06f90e6d96c47a2c57be409d2375829dbae24421ba8e" }, { "model": "cohere/command-a", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793114832-92TT9sxYluroj3R7ZM8c", "proof_sha256": "3737ffda762a1979f3be9648e9e7a4fc8cebecca3173da0e400a155913fd9219" }, { "model": "cohere/command-a", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793114851-eDTURJY7Ck8qBQP4atx7", "proof_sha256": "4310c630b0efe450c46fa81af4ed60a453ccd69e6318276c2db943de8e60dc87" }, { "model": "cohere/command-a", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793114875-2VpPFaNPc9g2yfqln2wv", "proof_sha256": "a4822c697b873442dd488449cee57254a59b689eae0f1b6fc2d14c84c7372a4b" }, { "model": "cohere/command-a", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793114889-s8n5bkehPetsKDKMWEVk", "proof_sha256": "02f292d497e3d88f6e3663e0f91e7b4ff987bf5d31f809afae310d5f4ec38e6b" }, { "model": "cohere/command-a", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793114984-Flp6e2wwwRawRHNvvN0e", "proof_sha256": "f3d66ee2803359e3efeb561c592f55f3ff840d212605dd7f6ce1c5efe1d9dcff" }, { "model": "cohere/command-a", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793115005-bbTZbwA1iRJ1nFLUgUFM", "proof_sha256": "15fd30b54f78d5e59ac2fa8e054346675df58bf4462db88c632245142d79ef74" }, { "model": "cohere/command-a", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793115047-4oiTcacch3AdpSatiOQl", "proof_sha256": "bec36c609aa3b83b379365d03411aa2cfa7e813c49db7be22129e53fda6dfbef" }, { "model": "cohere/command-a", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793115077-IGMTt2jUUbSIgJxCZWfq", "proof_sha256": "c13a4ebd63ddd84b317eac9139384ce81a5f98e29fefda415bf4c32921557234" }, { "model": "cohere/command-a", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793115139-EEqHexISC8MyeI7k9CtN", "proof_sha256": "cd3cd35843aff346d5cb82e0703a89fe9e59bef042ccfcde499bd3ab5d76f596" }, { "model": "cohere/command-a", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793115170-SUjF6FRusFSoIlJOMfC8", "proof_sha256": "96d97fdedf80b6a41a8d5c1057687b122dc87c194dcff87b9893d8900e4c25dd" }, { "model": "cohere/command-a", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793115235-b66Cp8J7cSKHQ1C0kZc4", "proof_sha256": "7b78e93396b1b5f556669524cf72908ec2da24aedefc6c7b5129bae6a7a58d22" }, { "model": "cohere/command-a", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793115276-rntBKtlNOomVsJXNwwSl", "proof_sha256": "1141cfe3fbd3c73b5ab1a86f2c95733cb13d7ad456b2b415c0a5e99a97060fab" }, { "model": "cohere/command-a", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793115320-7Iu7eJayMXZXiDesHpx8", "proof_sha256": "3f57b4eec9f8993cf99e427d9e852c01f560eabc6d16315b5b6cd8c9cdb69fbd" }, { "model": "cohere/command-a", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793115333-NypjoBSPqP7jaUUmK9j0", "proof_sha256": "fcf42a6e0f6fb1499cebdd50580a8cb146ab304568a7e8816276662a1f19690b" }, { "model": "cohere/command-a", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793115374-TqDE5DRE4AjJ3clngsGG", "proof_sha256": "a36c86ad939c466f8b0203c0bc09a0256f99f850ca41adfc71f94186c4330706" }, { "model": "cohere/command-a", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793115397-wrvT8dN0trkhjpwO4R4e", "proof_sha256": "00acf4b785a339c82baa7477545413d5a1b059f005856a2730daa9d8af66091c" }, { "model": "cohere/command-a", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793115452-hNV5B0cnfbLVDWARD1IM", "proof_sha256": "e99894362a9a2a1d46b8ecd165d2d78e0c779f83fd01eb2a59203266f68ea596" }, { "model": "cohere/command-a", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793115488-kJCG1kivviNaV9n46fyb", "proof_sha256": "b3e8aa0d7fbd645cd93e56285559b8b68b32ba051cddc0a5847bbe76b631a769" }, { "model": "cohere/command-a", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793115510-TmvrmvXiSsmz2tc2XD2T", "proof_sha256": "d388d6dc7f1a5bf363793479b18eb4ca7c6e3748e200c558d66210c2e759afa9" }, { "model": "cohere/command-a", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793115520-NjzKlvhagxuYf9eozIRF", "proof_sha256": "f0bfad604c742ce1cfdb0b3bbed82eb7292bff2add8384b5104e3a923be8b176" }, { "model": "cohere/command-a", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793115540-G5fdT9dzTQKLbSs1E6Gn", "proof_sha256": "f7532c34d13ddd1b46e12a23a28afad13c46fe5aa14322fe2b98cccbaafdb350" }, { "model": "cohere/command-a", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793115557-tG5hAflAWwPz3L0sZ40K", "proof_sha256": "bbc7a8529f95bb065ea872c8143b843c980e8cbb06832ae7509edf381aa3f911" }, { "model": "cohere/command-a", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793115566-JOCJCLCCrJMhutkHgkjn", "proof_sha256": "25ce3ef5025405856b9bef0cfe41fff9e7374f5018cc66766c0a0cb55d522f01" }, { "model": "cohere/command-a", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793115578-d7qGTzD8j1baFZtTRr5f", "proof_sha256": "bdb9aa7340c6eefdfafc2c365b273439ff2463f3cc212a1029744125fa506a3a" }, { "model": "cohere/command-a", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793115605-bc8Urcj2Ws35bK8VAYD7", "proof_sha256": "8056e8d6f02ef325b77805103ff969879813bdde83beb60aae2cb7dfcdb1eec5" }, { "model": "cohere/command-a", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793115644-muX7nOgsNVQcQbDu2z0Y", "proof_sha256": "ffdb6f2032adb99b6b70fe6f5a77f6c6beecde8cdb1dafc9f5af045c4c69081e" }, { "model": "cohere/command-a", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793115664-XDzKseKFwajw7uTkNNCj", "proof_sha256": "13fa0d9a58fb5a13988a5992ab3485cdfa86d14de50ef54ba317e2fc2fdaa585" }, { "model": "cohere/command-a", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793115751-cPfSJCbnKNNtFH1ioGyl", "proof_sha256": "edfa4280eea7227ef9823887fe98d1d34a96bd46dfa2be53524c1c5fab34bb1d" }, { "model": "cohere/command-a", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793115792-ipnCSEUhq45haAhyPCm3", "proof_sha256": "3bffeeb4afc11c1c584e40ae30fd34c62a70241cd6af7c1d0bf87fae817a9d2d" }, { "model": "cohere/command-a", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793115860-hicjVR6Nl4LoZAyCCBzE", "proof_sha256": "2f641010c4f0bf022361e883aa445c018c7641af764132f49679a2f58dd45315" }, { "model": "cohere/command-a", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793115877-whITdpV9yu2n2TLjBt7U", "proof_sha256": "31c80435b394ee1d0e5c6dca8d08d14c018c3dc2ceac8e5275d7ff8b33d3afd2" }, { "model": "cohere/command-a", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793115917-NpeNtwgfMZrmUAL3bIgD", "proof_sha256": "8dc5bfc27aa8e0f03428ccde4461228948f43ef951163ec68c65a113071cdee9" }, { "model": "cohere/command-a", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793115957-lDVNMBXMii83gM2UVtvu", "proof_sha256": "7d93076fb515b5f78956b91d84d8812b2dd118d17e8309cbb9356a451cfde470" }, { "model": "cohere/command-a", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793115996-NYuE3R9JdH450NWFvWGQ", "proof_sha256": "376856130c14235be76bf5c80217a43546e22ae47fe3a0a480ad3de707890ad8" }, { "model": "cohere/command-a", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793116013-pI0VJgtQMVWtHpXbkFDn", "proof_sha256": "170edbf5981785482211ebfc2a7c3caccb1f1ab286ab383e34e5764ef438c52e" }, { "model": "cohere/command-a", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793116040-jhvGGbjWCg6CF1qNgPry", "proof_sha256": "eaaf4d3ede9d56d824f2d4267937c5d28c3386039b386767c9f83e324188d192" }, { "model": "cohere/command-a", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793116089-FAXlezMh5S1r38XsbBxR", "proof_sha256": "6591e6abc34c50a426b4b7cf297a0c5fbeda7617f20fac6ba9ab2e1ebf62f004" }, { "model": "cohere/command-a", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793116134-5ofZG6iy7TWle9WQ38ha", "proof_sha256": "1e298242b51034ea5fb75aa31e92ed0b969534ef61c6b8361a02b3d943a3afac" }, { "model": "cohere/command-a", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793116158-BdwXV7tmGq9I8l76pxLc", "proof_sha256": "e35a68d079ddb248e907a17e350353c2ff5bdcd2b9c8b170a670710e08f867bc" }, { "model": "cohere/command-a", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793116175-v54gV8Vdnw9Q0mOFkCIz", "proof_sha256": "12db29ac4bc9de7e6328b5cca785c54d5a4e54c82c209b51a66ad34943a9fe8e" }, { "model": "cohere/command-a", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793116202-lji9b6ulbiuI9RbE9u2j", "proof_sha256": "450511414108894595db896ac7dff24482e6ea8c3f71e669e82ee059573b4642" }, { "model": "cohere/command-a", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793116266-Eu2nmZEATM6vogIB6D1R", "proof_sha256": "6b5c4be7ddc8ae33d2e1a6f37f7b73baf5a01470d966f29a301069eac743d5d7" }, { "model": "cohere/command-a", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793116308-oizVyVR0vKR8SoKwDzgd", "proof_sha256": "cadadea17723172f4c2f9640a4f8ac0f889a3cf8bf0bb32f29b15381c4c68ba5" }, { "model": "cohere/command-a", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "answer: j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261027T120517Z", "generation_id": "gen-1793116351-6TsgvAIlF7CRDaH1t6Qw", "proof_sha256": "e7c65ddedeb1b29fa74ac1239f7b7476f1732f2d86766bd34c23cfc182516f57" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793102728-ew8drCeZGe59I7uIUqcD", "proof_sha256": "b1b98e56e0ea375e3544c9391bd19dca98e8c8264a043aa30cb5ac24675f3e9b" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793102789-SsjklQjUybwadch65EP2", "proof_sha256": "80c2ebda2b883f616e18ff6adf8b431a2a549bc1aa6fc222a9a53589149deeac" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793102847-Q1uRTUkNMiDNDurglioM", "proof_sha256": "805211ee117d5bcfb563188fbc120d24848cd83b341163a6eec0f04cac807a93" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "- integration by parts: applying the formula ∫u dv = uv - ∫v du, where u and dv are chosen appropriately based on the liate rule (an acronym suggesting an order of preference for choosing u in integration by parts: logarithmic functions, inverse trigonometric functions, algebraic functions, trigonometric functions, and exponential functions. the liate rule is used to choose the u function in integration by parts. the order of preference is logarithmic, inverse trigonometric, algebraic, trigonometric, and exponential functions. the liate rule is used to choose the u function in integration by parts. the order of preference is logarithmic, inverse trigonometric, algebraic, trigonometric, and exponential functions. the liate rule is used to choose the u function in integration by parts. the order of preference is logarithmic, inverse trigonometric, algebraic, trigonometric, and exponential functions. the liate rule is used to choose the u function in integration by parts. the order of preference is logarithmic, inverse trigonometric, algebraic, trigonometric, and exponential functions. the liate rule is used to choose the u function in integration by parts. the order of preference is logarithmic, inverse trigonometric, algebraic, trigonometric, and exponential functions. the liate rule is used to choose the u function in integration by parts. the order of preference is logarithmic, inverse trigonometric, algebraic, trigonometric, and exponential functions. the liate rule is used to choose the u function in integration by parts. the order of preference is logarithmic, inverse trigonometric, algebraic, trigonometric, and exponential functions. the liate rule is used to choose the u function in integration by parts. the order of preference is logarithmic, inverse trigonometric, algebraic, trigonometric, and exponential functions. the liate rule is used to choose the u function in integration by parts. the order of preference is logarithmic, inverse trigonometric, algebraic, trigonometric, and exponential functions. the liate rule is used to choose the u function in integration by parts. the order of preference is logarithmic, inverse trigonometric, algebraic, trigonometric, and exponential functions. the liate rule is used to choose the u function in integration by parts. the order of preference is logarithmic, inverse trigonometric, algebraic, trigonometric, and exponential functions. the liate rule is used to choose the u function in integration by parts. the order of preference is logarithmic, inverse trigonometric, algebraic, trigonometric, and exponential functions. the liate rule is used to choose the u function in integration by parts. the order of preference is logarithmic, inverse trigonometric, algebraic, trigonometric, and exponential functions. the liate rule is used to choose the u function in integration by parts. the order of preference is logarithmic, inverse trigonometric, algebraic, trigonometric, and exponential functions. the liate rule is used to choose the u function in integration by parts. the order of preference is logarithmic, inverse trigonometric, algebraic, trigonometric, and exponential functions. the liate rule is used to choose the u function in integration by parts. the order of preference is logarithmic, inverse trigonometric, algebraic, trigonometric, and exponential functions. the liate rule is used to choose the u function in integration by parts. the order of preference is logarithmic, inverse trigonometric, algebraic, trigonometric, and exponential functions. the liate rule is used to choose the u function in integration by parts. the order of preference is logarithmic, inverse trigonometric, algebraic, trigonometric, and exponential functions. the liate rule is used to choose the u function in integration by parts. the order of preference is logarithmic, inverse trigonometric, algebraic, trigonometric, and exponential functions. the liate rule is used to choose the u function in integration by parts. the order of preference is logarithmic, inverse trigonometric, algebraic, trigonometric, and exponential functions. the liate rule is used to choose the u function in integration by parts. the order of preference is logarithmic, inverse trigonometric, algebraic, trigonometric, and exponential functions. the liate rule is used to choose the u function in integration by parts. the order of preference is logarithmic, inverse trigonometric, algebraic, trigonometric, and exponential functions. the liate rule is used to choose the u function in integration by parts. the order of preference is logarithmic, inverse trigonometric, algebraic, trigonometric, and exponential functions. the liate rule is used to choose the u function in integration by parts. the order of preference is logarithmic, inverse trigonometric, algebraic, trigonometric, and exponential functions. the liate rule is used to choose the u function in integration by parts. the order of preference is logarithmic, inverse trigonometric, algebraic, trigonometric, and exponential functions. the liate rule is used to choose the u function in integration by parts. the order of preference is logarithmic, inverse trigonometric, algebraic, trigonometric, and exponential functions. the liate rule is used to choose the u function in integration by parts. the order of preference is logarithmic, inverse trigonometric, algebraic, and \\sin\\left( \\frac{\\pi}{\\text{sin}\\left( \\frac{\\pi}{\\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right) \\right)", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793102906-cSOiqQmGm2a1XmUVSZYD", "proof_sha256": "ed5a2762bcd20a571e4f18e89f8df60c8555ca6d76425f4b5365275921829d4f" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793102972-515hRvhpTWUbhuJ6rlId", "proof_sha256": "a5d277caf52d862cf452ddd3ef7dfee562ac77882f18230c840c87628ede63b6" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793103050-4bRBMocCtmMYflJJbKy2", "proof_sha256": "e83cf5b4f9edc9af19da5ca362539bc27126f68356e62282a620890ac9b8c6d6" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793103128-NhKK87UrpcUk09MRGTLD", "proof_sha256": "2ccfd5609188bc59ffa28f53e25ee276517e19c544a8e56c3888991d30142cb3" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793103208-omqeDY9Dt0ozngIVEqWJ", "proof_sha256": "74f04f07ba9a32a3ae82eef286b20d2f9540a140075cbc38bf578ea92dd8f135" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793103265-22CBrevioUK0BMW2Kary", "proof_sha256": "c06e4854b8cd4f123ed1848bb72c605a2da51f6a0c434591ad6da40168c81fc9" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793103322-b71TEHIUHAJJjddM4p54", "proof_sha256": "c1a156238b5c96e44f0c9f1a7fe7fde4550b738fbafc88741cda0945f6647ca7" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793103379-7o3KErJPxdF9zQaVWk1m", "proof_sha256": "5ce97d377f886e983a424ebbf9f63cfd5695db060d7e3cb356cf2e41697281e3" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793103427-arNeiYpBFSpdTo9aUDzQ", "proof_sha256": "2f617eca933632b3d7487e36c32b6eeca1032887ab287358c25c9108eab02708" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793103475-dvXZSgZsFyQDko68zOQn", "proof_sha256": "cf88a6a866ccb41785feca6b3173e34693bacb7389226d17ebb274777117905a" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793103522-eEhFFM0wTeBD1XCYmy5A", "proof_sha256": "7740ab06b7650cff87a31a8da03d8fbc18058b01b3bc1c18afad451df59f51b9" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793103570-hdsrj11W2DcpDmhB5Ko1", "proof_sha256": "403398798a395d04f49cef9a44e7f6dddd41d2f2b55220b557294fd6306fecd1" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793103633-XrZtEN3p3pMRJsC6XUac", "proof_sha256": "67259214e5eec4a9166239f0655143d4bc3e4ca4b72b6ec759fe98426d4bc370" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793103680-iZsjK8yT41El1xqJmF7O", "proof_sha256": "cddfdb1547c04e92f3677de234473c086fc548d82b36b262cd4075286b94e8f1" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "\\boxed{b}", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793103728-lqXvXSf0QZ5RXhhV7qRC", "proof_sha256": "2017e9a54ff4f2a53021697286d34a068b000185bad67ae566dca39b05953543" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793103757-X2pozCFIt6VrMMvTD3ti", "proof_sha256": "d7b297f79fbdc89c4abb6c1dd47c8b712e709e9fe22d58ee60058ae727a1edac" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793103815-q1YaxEgDrgzI2gW8exFS", "proof_sha256": "2cc8aa171ef8da15986e6b92ac1136a7f7865e0623cadf2498f075ea4d187edb" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": true, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793122291-XynD76iAU8FEJDCkcDLZ", "proof_sha256": "4175093d1ab059ae3464cd98f163dcee85823a138b005cbac6f093d838d69881" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793113930-eHl4oKyk4n2hRrosPDr8", "proof_sha256": "1e07024dd07d83408ccf6708b066a6d47cd7b61110a33a17fbb7d203e8369405" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793113990-RjS0T7wULbYfoAH0ISof", "proof_sha256": "87792424dd0a8cf33b3727ce24485ae5a5da2402b3bfc656df53da1672c98f1c" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793114037-gjCCF6SuBmsZIhg3QFW0", "proof_sha256": "9215371a3ec161b54f9666a698dc3b4333b5e7cbe0e0be603acff24f335a7d80" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793114094-6tlOeyxFuWeWjwW0nBOI", "proof_sha256": "7d8072147c31ddd7148be6add748b9169b19b1059464f13bab9723ffc1620722" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793114151-CwuZcjGAlkr1muXQ3hgl", "proof_sha256": "43a6fa581671a535a888ea7e2548ad45bf28ca6d1164703e17ea06f96801f0c4" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793114212-NiMzLzD2NEMG6ldyk4HV", "proof_sha256": "f39a6507643cf94992cbc328fdc9f3b97a20fc5992ecd518e9518118ee4f8f01" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793114270-IoVTU36vEsa6YYBIU7eV", "proof_sha256": "c72540565d818419ca721146a04621a3dcb56871b1af15cf675320e5e75a7141" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793114348-57ZkFdBrx9usMY34MHMz", "proof_sha256": "909d4ea306480b5485bbded026c32b5c96309dc27a746ebec4961e5ea2d6c5bc" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793114405-u4SpPKw0oBPs2vebS2Tm", "proof_sha256": "26655339ba5539abe5b327dc7ac623ed9885c0ca172f18358f64eeaf7e0a4ace" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793114452-pmdfwofA1LyrcNIfmlDs", "proof_sha256": "3b6f696ff670ef808e42a29e6d33a307444b59a6c09754d993da07f1166ec530" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793114509-T27dyp1FFe3xEjtoN3Hl", "proof_sha256": "8b45ce29d69048dd97ae9ee6ccae53df292bafd7dd05adfce218b54e400e5172" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793114566-ShThtKWRSa53FZ0z9V4B", "proof_sha256": "daa263eb97215f5457d57f8c721d1345b9e5bc80f1a07ee703eaaf1aff4cc595" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "this reynolds number corresponds to the transition from laminar to turbulent flow in a flat plate boundary layer under typical experimental conditions", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793114623-k3jG8I6CyMcb6nUq9MPh", "proof_sha256": "07e766089dab6d5452ad527cb705967cc31fb54617eb2221789a97367eaf7cd8" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793114665-D6XDpmuOqMx3xAaQt2Df", "proof_sha256": "2016099ccaf67c61b7a4b2ea51034dfc2ab4d11128228025c7c78e4d687f6fe5" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793114722-2ZomIlMT8ovdON8EsYxh", "proof_sha256": "e3d4041bb7b418235aff3ac75a802e20741f040e2d954681cf7237acd264a812" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "- in conclusion, the presence of a small round cell carcinoma, which is a benign tumor, is not associated with any malignant neoplasm. however, the presence of a calcified nodule in the lung field is not associated with any malignant neoplasm. therefore, the presence of a calcified nodule in the lung field is not associated with any malignant neoplasm. therefore, the presence of a calcified nodule in the lung field is not associated with any malignant neoplasm. therefore, the presence of a calcified nodule in the lung field is not associated with any malignant neoplasm. therefore, the presence of a calcified nodule in the lung field is not associated with any malignant neoplasm. therefore, the presence of a calcified nodule in the lung field is not associated with any malignant neoplasm. therefore, the presence of a calcified nodule in the lung field is not associated with any malignant neoplasm. therefore, the presence of a calcified nodule in the lung field is not associated with any malignant neoplasm. therefore, the presence of a calcified nodule in the lung field is not associated with any malignant neoplasm. therefore, the presence of a calcified nodule in the lung field is not associated with any malignant neoplasm. therefore, the presence of a calcified nodule in the lung field is not associated with any malignant neoplasm. therefore, the presence of a calcified nodule in the lung field is not associated with any malignant neoplasm. therefore, the presence of a calcified nodule in the lung field is not associated with any malignant neoplasm. therefore, the presence of a calcified nodule in the lung field is not associated with any malignant neoplasm. therefore, the presence of a calcified nodule in the lung field is not associated with any malignant neoplasm. therefore, the presence of a calcified nodule in the lung field is not associated with any malignant neoplasm. therefore, the presence of a calcified nodule in the lung field is not associated with any malignant neoplasm. therefore, the presence of a calcified nodule in the lung field is not associated with any malignant neoplasm. therefore, the presence of a calcified nodule in the lung field is not associated with any malignant neoplasm is not associated with any malignant neoplasm in the lung field is not associated with any malignant neoplasm in the field is not associated with any malignant neoplasm in the field is not associated with any malignant ne not", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793114770-LDo3rH26lN7IN5MVxFCP", "proof_sha256": "5c2c972d6adc8db7fae4ff03fae3c6f8517ee302480079fd06015850cd252a18" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793114827-idoNQVQN7QwE5mAZU4EC", "proof_sha256": "4a23e7b56c91eb1363e41296ced0349ac21669354b754ecd232de25468d855c6" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793114875-DSectrGCNTuFc65dmKon", "proof_sha256": "2c1afc6de774921562ab2ab647d3c82580a2a3604dd9bce219b55350120540be" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793114922-Ors3rIDrZMJjsMRVxYXc", "proof_sha256": "8da16d78e3041610f1b53a3bb3b868d4102529376a7921601ebcaca80af66d0d" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793114970-twxF7sQw8957C3SoCioB", "proof_sha256": "0f522a036cee7c7adc5e8c2122fe7ed8f8900c1d50f686156235d85d29359200" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "b. culturing the corneal scrapings on appropriate fungal culture media (e.g., sabouraud dextrose agar with chloramphenicol and cycloheximide to inhibit bacterial growth, which is commonly used for fungal culture due to the selective media that favor fungi over bacteria). after inoculation, the cultures are incubated under appropriate conditions (e.g., 25-30°c incubation, depending on the suspected pathogens optimal growth temperature). after several days (e.g., 3-5 days), the presence of aspergillus species can be confirmed by the morphological characteristics of the fungal hyphae. this process is critical in the diagnosis of aspergillus keratitis, as it allows the healthcare provider to differentiate aspergillus species from other fungal pathogens, such as fusarium species, which also produce hyphae, but with different morphological features. in the case of aspergillus keratitis, the identification of the characteristic septate hyphae of aspergillus species is pathognomonic and allows for a definitive diagnosis of aspergillus keratitis. this process is critical in the diagnosis of aspergillus keratitis, as it allows the healthcare provider to differentiate aspergillus species from other fungal pathogens, such as fusarium species, which also produce hyphae, but with different morphological features. in the diagnosis of aspergillus keratitis, the identification of the characteristic septate hyphae of aspergillus species is pathognomonic and allows for a definitive diagnosis of aspergillus keratitis, as it allows for a definitive diagnosis of aspergillus keratitis, as it allows for a definitive diagnosis of aspergillus keratitis, as it allows for a definitive diagnosis of aspergillus keratitis, as it allows a definitive diagnosis of aspergillus keratitis, as it allows a definitive diagnosis of aspergillus keratitis, as it allows a definitive diagnosis of aspergillus keratitis, as it allows a definitive diagnosis of aspergillus keratitis, as it allows a definitive diagnosis of aspergillus keratitis, as it allows a definitive diagnosis of aspergillus keratitis, as it allows a definitive diagnosis of aspergillus keratitis, as it allows a definitive diagnosis of aspergillus keratitis, as it allows a definitive diagnosis of aspergillus keratitis, as it allows a definitive diagnosis of aspergillus keratitis, as it allows a definitive diagnosis of aspergillus keratitis, as it allows a definitive diagnosis of aspergillus keratitis, as it allows a definitive diagnosis of aspergillus keratitis, as it allows a definitive diagnosis of aspergillus keratitis, as it allows a definitive diagnosis of aspergillus keratitis, as it allows a definitive diagnosis of aspergillus keratitis, as it allows a definitive diagnosis of aspergillus keratitis, as it allows a definitive diagnosis of aspergillus keratitis, as it allows a definitive diagnosis of keratitis, as it allows a definitive diagnosis of keratitis, as it allows a definitive diagnosis of keratitis, as it allows a definitive diagnosis", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793115026-pp46L7cERyELrv6eFixA", "proof_sha256": "f38d4deffedcc52c2676612826de0e9fb4ee785b73b8234abd273364b98c20db" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "4. neonatal screening for cataracts: in many countries, congenital cataracts may be identified through routine neonatal screening tests. in some regions, specific screening tests for cataracts may be conducted if there is a high index of suspicion. however, in many countries, the routine neonatal screening tests do not include specific tests for congenital cataracts. however, in some countries, if there is a high index of suspicion for congenital cataracts, specific screening tests may be conducted. however, in many countries, the routine neonatal screening tests do not include specific tests for congenital cataracts. however, in many countries, the routine neonatal screening tests do not include specific tests for congenital cataracts. however, in many countries, the routine neonatal screening tests do not include specific tests for congenital cataracts. however, in many countries, the routine neonatal screening tests do not include specific tests for congenital cataracts. however, in many countries, the routine neonatal screening tests do not include specific tests for congenital cataracts. however, in many countries, the routine neonatal screening tests do not include specific tests for congenital cataracts. however, in many countries, the routine neonatal screening tests do not include specific tests for congenital cataracts. however, in many countries, the routine neonatal screening tests do not include specific tests for congenital cataracts. however, in many countries, the routine neonatal screening tests do not include specific tests for congenital cataracts. however, in many countries, the routine neonatal screening tests do not include specific tests for congenital cataracts. however, in many countries, the routine neonatal screening tests do not include specific tests for congenital cataracts. however, in many countries, the routine neonatal screening tests do not include specific tests for congenital cataracts. however, in many countries, the routine neonatal screening tests do not include specific tests for congenital cataracts. however, in many countries, the routine neonatal screening tests do not include specific tests for congenital cataracts. however, in many countries, the routine neonatal screening tests do not include specific tests for congenital cataracts. however, in many countries, the routine screening tests do not include specific tests for congenital cataracts. however, in many countries, the routine screening tests do not include specific tests for congenital cataracts. however, in many countries, the routine screening tests do not include specific tests for congenital cataracts", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793115074-q6lvoaqZAJWLqfQAa2Ee", "proof_sha256": "8081cbb4ec1f3d35c0efd8c2ef9bd80cd8878fb25f6d59bc007180e0efc1e69d" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "\\", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793115152-YKEDjgeNSf4kypFlCuZZ", "proof_sha256": "08a641dd80a0bb64de24e8910b3852f6a63d05027f19b3f4f5d9ca3bbc607e83" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793115216-95pK7JeHqtIW53Ve2ueM", "proof_sha256": "3126df3326973e485c506be47b51c40292c660dc8c1216ca9a39168da518646a" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793115273-gtmmM6F015l774gDo1gZ", "proof_sha256": "91d5d67120901ab0abebb41acb03dd808462f93c3804bdf51a0eaed4d2c89f76" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "in summary, the evolution toward smart manufacturing represents a transformative shift with profound implications for industry, society, and the global economy. while the potential benefits of smart manufacturing are substantial, the challenges and risks associated with this transition must be carefully addressed and managed to realize the full potential of smart manufacturing while mitigating its negative impacts", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793115320-zYCXvtIPTJHVtDaCgJKi", "proof_sha256": "2f76cf4f69349edfa1917af984afede3ab1f20e8fd1b59ea6b476c5030b9c224" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793115377-BlVS2n4HBFQBmLLyqb7i", "proof_sha256": "000eaa895364e2cc1ebf622cebcaaf328ddac64bec0d298453ee72e2fdcea1c0" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793115424-eo2bbxSFbNscKZHU3qrn", "proof_sha256": "dd594e86fe1540f93a5742b8055aaf42d03103a9a0bb0b58cb0989e87f3c4bac" }, { "model": "rekaai/reka-flash-3", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793115479-81HqLCxwrudgSesx6qOG", "proof_sha256": "0646d44ab230e739986e0d249ff5ec39f036b4491612ef38fcca8507b0ac7515" }, { "model": "rekaai/reka-flash-3", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "conclusion: the plaintiff has presented sufficient evidence under each element of negligence. the defendants actions caused the plaintiffs injuries. the plaintiff has demonstrated that the defendants breach of duty directly caused the plaintiffs injuries. the plaintiff has presented evidence that the defendants actions caused the plaintiffs injuries. the plaintiff has presented evidence that the defendants actions caused the plaintiffs injuries. the plaintiff has presented evidence that the defendants actions caused the plaintiffs injuries. the plaintiff has presented evidence that the defendants actions caused the plaintiffs injuries. the plaintiff has presented evidence that the defendants actions caused the plaintiffs injuries. the plaintiff has presented evidence that the defendants actions caused the plaintiffs injuries. the plaintiff has presented evidence that the defendants actions caused the plaintiffs injuries. the plaintiff has presented evidence that the defendants actions caused the plaintiffs injuries. the plaintiff has presented evidence that the defendants actions caused the plaintiffs injuries. the plaintiff has presented evidence that the defendants actions caused the plaintiffs injuries. the plaintiff has presented evidence that the defendants actions caused the plaintiffs injuries. the plaintiff has presented evidence that the defendants actions caused the plaintiffs injuries. the plaintiff has presented evidence that the defendants actions caused the plaintiffs injuries. the plaintiff has presented evidence that the defendants actions caused the plaintiffs injuries. the plaintiff has presented evidence that the defendants actions caused the plaintiffs injuries. the plaintiff has presented evidence that the defendants actions caused the plaintiffs injuries. the plaintiff has presented evidence that the defendants actions caused the plaintiffs injuries. the plaintiff has presented evidence that the defendants actions caused the plaintiffs injuries. the plaintiff has presented evidence that the defendants actions caused the plaintiffs injuries. the defendants actions caused the plaintiffs injuries. the plaintiff has presented evidence that the defendants actions caused the plaintiffs injuries. the plaintiff has presented evidence that the defendants actions caused the plaintiffs injuries. the plaintiff has presented evidence that the defendants actions caused the plaintiffs injuries. the plaintiff has presented evidence that the defendants actions caused the plaintiffs injuries. the plaintiff has presented evidence that the defendants actions caused the plaintiffs injuries. the plaintiff has presented evidence that the defendants actions caused the plaintiffs injuries. the plaintiff has presented evidence that the defendants actions caused the plaintiffs injuries. the plaintiff has presented evidence that the defendants actions caused the plaintiffs injuries. the plaintiff has presented evidence that the defendants actions caused the plaintiffs injuries. the plaintiff has presented evidence that the defendants actions caused the plaintiffs injuries. the plaintiff has presented evidence that the defendants actions caused the plaintiffs injuries. the plaintiffs actions caused the defendants actions caused the plaintiffs actions caused the defendants actions caused the plaintiffs actions caused the plaintiffs actions caused the plaintiffs actions caused the plaintiffs actions caused the plaintiffs actions caused the plaintiffs actions caused the plaintiffs actions caused the plaintiffs actions caused the plaintiffs actions caused the plaintiffs actions caused the plaintiffs actions caused the plaintiffs actions caused", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793115526-Kg33o0wtk51OWxKpCK5S", "proof_sha256": "e046f20575c7d8d4d0a08068366382bd8fc1440f0317d478e3dde72777b21c97" }, { "model": "rekaai/reka-flash-3", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793115583-nDDjPOR0TB3y5aGqDU2L", "proof_sha256": "f77e7377a021203632c4631f0b955a79ebd6a85fb31d7350fcc10564249944bb" }, { "model": "rekaai/reka-flash-3", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "the defense in this arson case employs a multifaceted strategy to challenge the prosecutions evidence and present exculpatory evidence. the goal is to persuade the jury that the business owner did not commit arson, and that the prosecutions case is insufficient, unreasonable, or legally invalid to result in a conviction", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793115640-07aYwoY8661lYagSxonw", "proof_sha256": "e3be06aff1b887d8b2b848624d447437c21b721c98033094701b191edaf90306" }, { "model": "rekaai/reka-flash-3", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "\\boxed{the plaintiff must prove that the defendants medical treatment deviated from the applicable standard of care, and that this deviation directly caused the plaintiffs injuries and resulting damages.}", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793115664-CHxpFqv16AFrmpfx9Aze", "proof_sha256": "ce0456a128971d087ada2fde88a7132583b3d0003a39bf3e553273ab948d3481" }, { "model": "rekaai/reka-flash-3", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793115700-uZpVScahxsjklflnZuHW", "proof_sha256": "7730ab7da8adef8470976293d754931828501eb86ed5d99d445769bc8015aad3" }, { "model": "rekaai/reka-flash-3", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793115757-CLU3O0syznfQ0ezytt0S", "proof_sha256": "31c00fb08aa495c1f8546fe17601902c2e84c61f54edf03d51be4fa8924ad391" }, { "model": "rekaai/reka-flash-3", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793115814-cb0Wi9hidH76zph6L3oQ", "proof_sha256": "273db18a5e9d199d5165862436bd78e17a272319e16bd70a265bc439c6d42869" }, { "model": "rekaai/reka-flash-3", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "in summary, dark adaptation is a crucial physiological process that enables the human eye to adjust to low-light conditions, thereby improving visual sensitivity and detail perception in the dark. this process is essential for activities performed in low-light environments, such as night-time driving, camping, or stargazing. additionally, understanding the mechanisms of dark adaptation can provide valuable insights into the management and treatment of various eye diseases and disorders that affect visual function and quality of life", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793115871-jDIhduq6fdGNF2LGKE8g", "proof_sha256": "af22ff6332b5653f2a0dfd6743e87c833c283e5f4c99c34a69e733bda2fb5e7d" }, { "model": "rekaai/reka-flash-3", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793115918-Y211N06mfh9yK2XMOvZs", "proof_sha256": "84a524af609c3db8e9240f2d47e9fb6a22b25f8a6e5e815497757fd2c60c2b8f" }, { "model": "rekaai/reka-flash-3", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793115965-aP0ZJwknMEbPWnoFfGjA", "proof_sha256": "658d55211d878722c852db570feed58560aad725b059122de5db510aaeeb4de4" }, { "model": "rekaai/reka-flash-3", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "in conclusion, u.s. defense spending has historically been influenced by a complex interplay of factors, including technological advancements, geopolitical dynamics, shifts in political priorities, and the economic context within which defense budgets are set. each of these factors has at various times significantly shaped the trajectory of u.s. defense spending. understanding this complex interplay is essential to comprehensively analyzing the historical patterns of u.s. defense spending", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793116010-OtZXSyk20ct3uYGKCcRZ", "proof_sha256": "229cfcc446f104724022536e285c43c8527e25fa2974174c31a0e24911156019" }, { "model": "rekaai/reka-flash-3", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793116058-uYOteLmcOvIxTUp9JvBt", "proof_sha256": "6ee54ac8f76b19c24241b0e00f9966cbf37f1fa685ffe5de1df6d52eecf68ffc" }, { "model": "rekaai/reka-flash-3", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793116106-WGAcf5PXNal08QAOZJtE", "proof_sha256": "a49aa4dea4bea8df5c58182c62f454f10ad1d0539cdc675184312a1c4cddf6d4" }, { "model": "rekaai/reka-flash-3", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793116120-fbd3QnuUYQqTRgJwq4IO", "proof_sha256": "f1f0766de6c6c2973b2a83a2484c9c1d9b6484779e6cc78be69c817fe8cddaa8" }, { "model": "rekaai/reka-flash-3", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793116167-2bX8IueW2GMUxsbA86nf", "proof_sha256": "633ced47affa76d4d8a20ee6565cbb0c8fa677688c28968a2d04a0fe10fa9262" }, { "model": "rekaai/reka-flash-3", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793116224-Wf2f03hHtUdjaS7UuNJ3", "proof_sha256": "e2ef9afaf052d3eef9eb75d1338a8a79ac387cc18cd43ccb591ccc70a28951bd" }, { "model": "rekaai/reka-flash-3", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793116272-W5bWOWAQl874ZaxatK5a", "proof_sha256": "34ac193910cc795c685a31d15ed29c44220de83f8338965728c12b093eae0f73" }, { "model": "rekaai/reka-flash-3", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793116350-VPdGgWxu0NB18XwSmpw2", "proof_sha256": "59779b0ad871e5f50e2b55803520d0cbde71be36dfd64ead356f8b56acba9014" }, { "model": "rekaai/reka-flash-3", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793116407-NUrQZsG2DQmKKcaAdzj7", "proof_sha256": "5dde32b17e53a087b1ded014f725e9dae3ffe49fd9603e84f58de3acbbf4e438" }, { "model": "rekaai/reka-flash-3", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793116454-Lj8Wf62k60WELs9InBI7", "proof_sha256": "6f97c716aa8caa8118e53f25a592b5fbacef96d2df97bfb34a1f269e1f7d6844" }, { "model": "rekaai/reka-flash-3", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793116532-u8vLeneApEpqwhWE3Ubj", "proof_sha256": "2af1258629399d1867937a03aa9cd2c340d2dcacb5b3980da46d5040c0a9e1a8" }, { "model": "rekaai/reka-flash-3", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793116580-Ou8Ob6UdKFpOq0CdooA0", "proof_sha256": "7dce070585875f84e66c805f745b1a486aa59e6a19be48f6b4406037b17ddf9d" }, { "model": "rekaai/reka-flash-3", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793116627-UNOC6zFVPrabokrIQ6Ov", "proof_sha256": "f0dc536207fd9cfb576cf3df458deb207a4b75323c9756e10132bc1490b5dca2" }, { "model": "rekaai/reka-flash-3", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793116685-DYIoDqaq1xPozL8wJxK4", "proof_sha256": "2cbb8336de21a52e95e73d4a8886ee03846cd5bbfcc59e16526d027933156fd4" }, { "model": "rekaai/reka-flash-3", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793116733-o5GTwE3Pn1Npm1CFTEI0", "proof_sha256": "a6098b4c2989d9f12189843e11d9c927ee1720e1f4b793bd00764d93fc28a456" }, { "model": "rekaai/reka-flash-3", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793116780-IOT7ZAmszxzw4pSY66Ps", "proof_sha256": "2a6ffe63ba8a8f8a556a3ade010487f3f7c4824509542c0e79f91a46dacb9350" }, { "model": "rekaai/reka-flash-3", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793116837-vcw2XPsW4SANUomaEFAF", "proof_sha256": "9caaac4a96cc1db56100b7d166b95492e6f5f55416bbdd8411bf28f1ee5b876d" }, { "model": "rekaai/reka-flash-3", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793116887-0UAn0rIF7PHQQMNODLrk", "proof_sha256": "f3f702f08f74b5425ce51a8cd50b6e81bd8dd79d0acfdf057c1fc48cdfbeb3b1" }, { "model": "rekaai/reka-flash-3", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793116936-GzsdlLvoFwo9nMZ7yEta", "proof_sha256": "04c6146f5e528d35b524949fee68b6f6cdb578750d66c789d8bddf46de814d74" }, { "model": "rekaai/reka-flash-3", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Reka", "run_id": "20261027T120517Z", "generation_id": "gen-1793116997-EY4YVn0yOmHeaxDn67tP", "proof_sha256": "493ae63ef362d82888c26c5d6c3535a49e8ead000feb342739ee774dfe746a53" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793102717-6u73nfKuURodn77lJubs", "proof_sha256": "891c893071be8ec9addd701e8caf9bd9f8f0c58b84a862017ad09904fdc48f76" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793102827-aREjyMIP8SwA8824M73R", "proof_sha256": "e21f14105d487958871923c8b4d7b7ad9b3a68aa991a9886ff2374a952bd1dd7" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793102916-BagVvlHtARqQzBSHRBIl", "proof_sha256": "59517e0ba3fbe82d82f390d977b2ed5da91cdd8ad1a0a54ce2d48af6dcc0111e" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "the best answer is b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793102989-brpjXQonYQKQK3Dm6Xuu", "proof_sha256": "5ef6f31a2dfadfa3cf0121c4e955ae9231df0a50c50b459f4a6614d4e12b68f0" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "the final answer is: $\\boxed{3987}$", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793103106-EeQmYQGC2C8BeSuorg1Z", "proof_sha256": "615a9f1db390bffed85e0a57066a6a7ca6ae3db38635fd62a132507d6c62ecee" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793103232-F3moW4H9rhm8GCpG9gaz", "proof_sha256": "de18a35803be282601b72585b31fdfa10deb03b029f3fb4557bf4b9f3312e7cd" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "the best answer is d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793103320-Zqlrnzm8I7munJ0w8ReO", "proof_sha256": "ef2e4cf02edb7c4cc315213e672d6bf3835dd3efcc49b1bf16d384cd8ed930bd" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "the best answer is b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793103407-u9SkJb0hfGC8vcCiN7hw", "proof_sha256": "570d5557412f2cd4c123c9afa7ef79548eb46b8a2fcb4daca1534090ca7bfffd" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793103470-1bL55A4UYtv8DELZZT88", "proof_sha256": "d79a2ccf83903c540cc58081e630846cc3887ed6d7282b084f195489eef85ba2" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793103544-owYE5OLAeAQC0miYcelC", "proof_sha256": "163b92ff8775121668bf905ea824b2e311737bfc6337b09c72d1e09f565aad1d" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793103656-UWNwvLCYz8bgyc8ZdhK6", "proof_sha256": "901fee2bca9d36002986ef691acf947c9b67fd91e483d5f85016ba72406d487d" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "the best answer is b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793103732-VUGFVlqjqsyg8JyA1yz1", "proof_sha256": "5bf4934f6c40c23884d31e4ce53eba096739a3c31cd32ae749285335925f9bd0" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "the best answer is g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793103802-sogpg9ZYnukQl7L8YXOY", "proof_sha256": "259e35b0e1473130ff16532173684260b0c8380e35e0790c08a20890638453a5" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "the best answer is h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": true, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793122291-FLxDZWQlN6ek52pkARdp", "proof_sha256": "d6ddeed9c59830a0a07ca22803ae6f2b4c39abaedbf6f08ea3d0c5f8dfa88f4a" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793113930-QY3T46j14s8mmU3JSxXD", "proof_sha256": "14b858d029f2617f48d513d241b6c18ed5883c615b8f722a5c856bde17008d3a" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "the best answer is b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793114002-WjcalCWyvNBhs9VudBZT", "proof_sha256": "6def62119e67946b0f396e312b14286d849f078bebbc4f3497eab6bb2e411992" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "the best answer is b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793114106-dKIm8fQbMLab9N2pRM97", "proof_sha256": "4d1935c67c4076983e633c6c7ff38896afa8b2ac2f0c8c4886aa5f8894b6b217" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "the best answer is i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793114166-tL1RQdQVHPhFcSinbV2c", "proof_sha256": "465a02cd9e22947d6028279b3fcc0fe95e144fcdf4b3a008979097d5e23cd9cf" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "the final answer is: $\\boxed{e}$", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793114231-xkWjmHknQxG2KELPZPCW", "proof_sha256": "e5ac87e190916d64c4c72d90cf9c3cc30bb7967d3476420213c41116fb379a18" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793114291-hNHYVRkuwegNPVx1mwCI", "proof_sha256": "61debdad43e55481303d9c55dee6bc3e3ad7388a23cdd7ac3a8975add0d26158" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793114330-gtTlUl65zW1zWCely0mF", "proof_sha256": "4dacd8d07a0e9478528acb17fb5695ead5e14cf9908254602acfa175c870f5d6" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793114371-b6DkEBYZsdmNSbkd373p", "proof_sha256": "d7128b8730800a84a1d7811ec7cb9444a21104daeaef34c453bd21df37410e58" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "the best answer is j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793114432-xIS4xkH21QVlTyAAFFQg", "proof_sha256": "4fbdf1855b2c9949bd2b2dc30b09942ee155f322cf80358490979ec103d05b8f" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793114505-g7KNz1gnc5OoAfTXts3k", "proof_sha256": "fdc75c618c61f36a0a7428324f08b345a6fa2a85e028792ac8f7f2e3034752bf" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793114565-vg4u6u8Mynu6c2qpAoJT", "proof_sha256": "504f716ef25341055d00279046b9de2b3abb681100ac48af0afc8b1326746bef" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "the best answer is c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793114616-sz2jQOG4My6ToDSORcqx", "proof_sha256": "537f142998f2b41a5ea1845d06ed95e08da43218212c7b587874d2c65787259f" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "the best answer is h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793114674-4xYhc7EeWFHO5G7RnMM5", "proof_sha256": "15ad6568591cbf10eaece67f481f9a46036c5de217c0dd66a5beff991cc61d3e" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793114749-992ApZhUK70orMYEe6OV", "proof_sha256": "66b26bcb1d69c01dbd2d7ea91db45361f5ac055938a5c7635fae7487f94a5b96" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "the best answer is d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793114813-kRw0O42POYw2IAcBNtOA", "proof_sha256": "f382ed27151099b65f0dd19f62549c9087d31fb84be3d66b911923c1e2dd2da4" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793114878-XHF8xzBOeNJRJWb01TiF", "proof_sha256": "b50359212f437ba285bd7c1bb2863a8c5e43a4d6c5f480a1828df4cd9606448b" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "the best answer is b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793114928-yxIBSaihOUbU0M3du4F1", "proof_sha256": "d0d9f4143a96064c89b15435a3da40fb555164f2da434ed1f02682e012d5437d" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "the best answer is d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793114989-4fpTKeUMUGiQs65Gqexk", "proof_sha256": "5ec5f5f5ccf4b5826cd668861d250fdcba906be5d4d26ceebd44fb1a8b0916ef" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "the best answer is c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793115074-SmxevqgwthYVtfSyutJL", "proof_sha256": "2873996cd765861aeadc473cb06a5e156e550a7ec7a171a38b23e2e6206b0d8b" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793115140-hC5ja7lfdVgTTZkIIz4V", "proof_sha256": "e76e226028e7081252616646be57b5b9a6d2470aaabefb18a2a10538f293c233" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793115186-kKf0V1v4KZbMFaKX3MLs", "proof_sha256": "09debaf9d6be4c7c13e64b0893b20f2da1fe5c5f71ac1336ba04c561f60c6fda" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793115268-n0ebo359mitYV8vWUOvx", "proof_sha256": "4e257422194bbca8e5d1c93e56c97dea6bdcd184673518b61a7eced56e1bf4da" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793115318-YEhLltrKUTUPH5p5iLMQ", "proof_sha256": "7e7e8673eabc47a88405a653222b32971277f566ce35d4521b7c5f1245261e10" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793115357-Ay7uefkukHX7qa6gcfWN", "proof_sha256": "d4deb1543fbfd3c68bc4f54a7f8b1a78c5bc332d7ac2f19b803215677b8d4f27" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793115409-DbY7yQkCjXS6bQbB8vp3", "proof_sha256": "74216e97b27ffb9149a2c906b81cc7d3e33c755794e98f7d590ad1376d17ade6" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793115439-9cXBoRIIONf7QDJQWIun", "proof_sha256": "3ab4caaaf0d9f15f31ddf22df4b58523372df455fc1a1a72b33c1413e9b4dcc2" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793115481-OJtXoGNIhhUtCYy1kdn8", "proof_sha256": "e0d678c2190bad32b0ce376d42816df865bd35589b9c2aab1c6d89c9b1428518" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793115530-8jy4ZvM2GsnMTKQLlqgK", "proof_sha256": "afe4b731b393a462a6981a94a3316a376892e924b6fefce5a76a5ecfcfc43e04" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793115561-kk77Ih74wm30e1PhCLBz", "proof_sha256": "8246f1661483701d03c8d82cb063011c2409da3e8d52bba0aea2f94d903299f3" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "the best answer is f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793115601-Za9JeWrfuNtVtwd83ioJ", "proof_sha256": "a87591ee2b8c5e37990fb329807602622d61225e653a3ed7b153278f6906e9be" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793115651-vxrAHsCWapmo41JWuEO9", "proof_sha256": "a171a80e11daaaae69567a3d755edfe85171dbc943bbbeb29a98515af24b57e2" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "the best answer is d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793115710-CiT9nnTyknKTM9FeAyWs", "proof_sha256": "b48da7324a4413a1e452a6d50122cc2dfcf3a4833a5a21b898d4844ff9450161" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793115752-xG7h9Z7fNqhzUI06NjWI", "proof_sha256": "992f2de25eb15fee1a3a8b19ce0651bd798daa2fd7c6a553bc3de9102d33f776" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "the best answer is a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793115811-jfVj7CBrbEJR1cQEZnTs", "proof_sha256": "39727cdede3cfb0e5050879de45820f40e251481874b601b32912c9c63aca6e6" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793115867-AqBt2o4fdGbOER0aXEO8", "proof_sha256": "2c99c889bbcb3ecd27cf3e9652a161abaaec89886954d32d1b5a4a1cf0f19e0a" }, { "model": "meta-llama/llama-4-scout", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793115898-AIVAoZ1xh01IcZtEgJxU", "proof_sha256": "ebf2635d838edbd596f948d1e295c4ea90e85dea5ebd31e6440e96c761b24068" }, { "model": "meta-llama/llama-4-scout", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793115956-bR4SdaWHG9pexBuRMW9s", "proof_sha256": "6754ac55ef153a21cea2519df7faf6f7316613e988e4504e78bf127a65ab7c18" }, { "model": "meta-llama/llama-4-scout", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793115993-YqjfiTJK4WiPH5mWc7im", "proof_sha256": "9e3793fab881e37bb45650e43fa84dbf5810a629a000bf076005501433f7f55e" }, { "model": "meta-llama/llama-4-scout", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793116044-Oqmo6P1S3WY6XVwiazg2", "proof_sha256": "dc71df8461f3d6925a4256734f7195425f3529d0396a4e1506eacd5d76c0ff3e" }, { "model": "meta-llama/llama-4-scout", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793116058-St6O0lKdMlJkocx37LkI", "proof_sha256": "07910ef0a9080663c60b1eadda495b26709761aaf0be1ebf4dc9016df935bb64" }, { "model": "meta-llama/llama-4-scout", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793116072-wxMrrf0YSzqgeqJNoIqA", "proof_sha256": "b1f4681f72338f23df10ccac15cea374a0b72de3a443e0463f68b0dba97d1015" }, { "model": "meta-llama/llama-4-scout", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793116134-MAOtboK08521XU2gbxYc", "proof_sha256": "577c36d7f79700d0e3c531c855fbdb36de3357844a5a96f17d5fbcc1fdca1c2c" }, { "model": "meta-llama/llama-4-scout", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793116175-1rbVl545qdDhEI7LehjT", "proof_sha256": "cad54023b4145d3859fb6f828551d39a2d2fd26c31bdaa751149dca5afb9f3da" }, { "model": "meta-llama/llama-4-scout", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793116202-llO7jAM0akJYEwKSYdE3", "proof_sha256": "14c1f877cc4e259f1ba3a743d8a20ca7bc0ad67ec4bf9dea06f88acd4127007c" }, { "model": "meta-llama/llama-4-scout", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "the best answer is h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793116260-jfKWLFEnZnwy8UKPLBNW", "proof_sha256": "7d0a906ed1db821c1846c8f13095fc8e3f7624adce77cda62472b62e56ae5d6b" }, { "model": "meta-llama/llama-4-scout", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793116293-O6fOTezMgEdGKriS0ShV", "proof_sha256": "3b49a4d6548a608186c51d3c0461847c6f36cd9068f0248d73940b660776d5e6" }, { "model": "meta-llama/llama-4-scout", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793116348-7WdlcaqAbgr96ilhkPSc", "proof_sha256": "976376bdf0eeb2dd1fb3d9b2418478a224f98d3e01b6590c58d9f2dd2857f985" }, { "model": "meta-llama/llama-4-scout", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793116405-7tXwhD97f5ozH9ulJuMF", "proof_sha256": "d002dd3552a60f2d1bb7b85aa9e0edaacbd5d7b769ee91af5fc6afcc40bfdd45" }, { "model": "meta-llama/llama-4-scout", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793116426-zB8n4iIjEYDrtuaHWT2L", "proof_sha256": "7fea0c7666cfdbbe74a03c92529606f55f46f79c63245edc13746e3ef7bdca1e" }, { "model": "meta-llama/llama-4-scout", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793116440-W0YrhF69f5QEeNPXXSJw", "proof_sha256": "f3b02e244bfcdd88ce6a769d6e8223cd222bcce86f4f4102125d46448da02259" }, { "model": "meta-llama/llama-4-scout", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793116463-M33UoS7cmGgFh6Tffzs7", "proof_sha256": "01b6d5b148f7b69d12d7d3733ca535a80d2d02aef81fc6290de51da392fbb0ea" }, { "model": "meta-llama/llama-4-scout", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793116510-QL1kLyumzlXggREYZ9Zf", "proof_sha256": "5f21f6d589dd869901ecdd0205a277e9bfa9c2c30b4546ba81a30bdf94093735" }, { "model": "meta-llama/llama-4-scout", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793116524-i1w3QyqzFOnHFxp1IcKL", "proof_sha256": "aebd90f15352a99f09787a78f4bd5eac6d820b2989e88df82e8c79d09e0a460e" }, { "model": "meta-llama/llama-4-scout", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793116557-CxyowlonOXIWZrCr6ded", "proof_sha256": "a62eb84e9a52c0e7eec528574497e87e320d98bc5df5688a2efab12ee30ab28d" }, { "model": "meta-llama/llama-4-scout", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793116594-FEv9HC62nRJgT5OUrdT5", "proof_sha256": "2cc3b40c968583bfc81b76e78e905ff3087fcb90f874a658bc01570300f1cdc9" }, { "model": "meta-llama/llama-4-scout", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793116650-O4DgT9XJz4dawygFVYBU", "proof_sha256": "958efac336fe513a28b6aea85dc7dcbbd1c4b87d6dc542088b99e11a8f1b2df1" }, { "model": "meta-llama/llama-4-scout", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "the best answer is d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793116688-uUS7Bpba8RZvI5mTIxTQ", "proof_sha256": "4ed2e922cfda742f2f4364afd0edfd3908db56ba976695557374ba1df84e371e" }, { "model": "meta-llama/llama-4-scout", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "the best answer is a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793116717-PZmi8BNi69RsQE1xUgBR", "proof_sha256": "645c2a96a9edf73b3842132aae3db4292cfb33b32f7e36faa2767bb8679da1f9" }, { "model": "meta-llama/llama-4-scout", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "the best answer is i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793116789-t8c0zADS0oH29fyLLsra", "proof_sha256": "bb92e01cf62e9e865ea0cdd26cf327178b3f73a06180752b023e29098347e9e3" }, { "model": "meta-llama/llama-4-scout", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793116842-sTfo1HzNhqEqdC9ovoSJ", "proof_sha256": "a55f4c3d3f0fee2fa254aea6010a6f332c7231640649dfdfbcaa10dfc2cd483f" }, { "model": "meta-llama/llama-4-scout", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793116877-RDQXYpY6NbzrBMnpkaFx", "proof_sha256": "6bc8584c501846e8e32c0877ac77850201e29452afbe17a1699922ea316bef58" }, { "model": "meta-llama/llama-4-scout", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793116943-dBR0Q0HjFjNvOSy7sA2H", "proof_sha256": "b88a8590738ede61e5ec3ed150b5e606278905047c6d677d3d30270f02fd9d5e" }, { "model": "meta-llama/llama-4-scout", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793116978-EiupxvG36FPbnEShlJsm", "proof_sha256": "2218c62bb833bbe9779727ff6539aa68921308a56121b342fae7584670e0aff8" }, { "model": "meta-llama/llama-4-scout", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793117026-TT2wQdNB21fLpY3Lf4pM", "proof_sha256": "7881ef7987b6aa1fc8aa869b9659d4fca4da49c8c75e2136c4fddf26d50224af" }, { "model": "meta-llama/llama-4-scout", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793117059-jKa4jT2Y8KpfjuDIaUz3", "proof_sha256": "731f5fcd5bec960c55890c87a2fdc457e0cd6891fc5e4e18f56849622f67a7e0" }, { "model": "meta-llama/llama-4-scout", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793117097-rILvx80zOFOQU33jr50z", "proof_sha256": "e2ad9e8ca41b5658ae56e7ed2b13e5458c8216231c24a054bb5a1544e73b9538" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793102717-3eDLfQxgEuHOhDJL5AQS", "proof_sha256": "5203e34b18776561eaeed31319e7a10f97b97e266a3a8cdc15cb735a1ef06ee1" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793102799-DmUhPpvZy3lkkQSOFxeF", "proof_sha256": "eb090d489118d6777f24800920859cd0107dbb369ca7342769148e16e96d12eb" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793102901-2TRnkVLj1RqbjS742OSA", "proof_sha256": "366b7caddf00f7b27d699e6e45fd8786e0494c80571281c731b1c10ff9596cd9" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793102968-5durD9Rg6BqzKbMex7d1", "proof_sha256": "aded0470c1490722175843b9d6da8032aa2e5ed2ee2cf45458f8311138cbdfbd" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793103050-ukXDrS40S82yXWVnlJwq", "proof_sha256": "a6d35d77dd3014487e95b9524cb4e7c70a381a2354649e53d0eb2c3e15274c1e" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793103153-LT7wl4SvK9eANWckwN0y", "proof_sha256": "e7c7ef541ac540668c72cded763a595c4a783533f5e3f2cbacb50efa729de624" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793103229-Vnquvd85diSjjJTJo4Ga", "proof_sha256": "3986ca404bbdab1e4df21006a9c07029f74650867f8d36a6865e8756b283f969" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793103296-xDVIxsclcy6v5o0wrMis", "proof_sha256": "765577fa3cbeaffebf5a1b40b999f1da0e79a3bb569e3f7ffce09decdf2ec378" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793103384-J0eYsvDotpmcyCTe9fbD", "proof_sha256": "8a7654a820098079a3801a13836db31e8cfda44f5c73c0996da961dedd53bd7f" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793103465-SF9sQTHSXzrFnsVdhO0b", "proof_sha256": "d841bd760c97b64f361e3c62667105b404dfecb349c15921fb1d7af8f88f5b13" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793103537-WaCKaiWsNyhSPyx4jLSq", "proof_sha256": "ce42782f869ccb39777bf903d387a987947d4163ed36a4aa766c5d765f7b725d" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793103567-OSdCAzMKstDR0woGyw0w", "proof_sha256": "2f2a3ca606ab4befa36d5f0512707333f2eabc554a7d8265cacbd0e51e5febc0" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793103640-ZutiUOQ8ApNeeWdwHFGH", "proof_sha256": "d24e72f7fe9eb853b304421f43e99b69a88c8c0bf5b1601fad5d2324769a2e24" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793103691-XZc0N11ruz8yHobyikxe", "proof_sha256": "d0206691ba9c91e73e73fd0c3c7e5c7d4b9f894842b93e0e976a100f6d0b1426" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793103763-8Q5nfCFW0VnhGeuDKwhO", "proof_sha256": "b1e7c1fd312668b60367f9903e019707d7ebfb65d56e8f69bdf30f5de86ea662" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": true, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793122291-EsJLXdhmrH20qrarNkyl", "proof_sha256": "597ff0cfb8c9fbaa205c6b05c45ac94dddb24662356497e1b8f40b2e3567a61d" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793113930-d8jyjqXEhK0bOFQWa1Mv", "proof_sha256": "52931eff855c49077a6999d6a8d175dab2d66024993057b17b2d372e8921fde8" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793113988-ZqWvWaiAY0f2FFN1CJG6", "proof_sha256": "7899a0dfeadfa88aa93962c344af1b1a2c10ccbb2729ffac575645af6d3faedd" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793114038-YsQNoic69ubXQbqojvFn", "proof_sha256": "85929058d61ed09804e6812fa3124480729aa402a3be8ae79c54e5b81a5d4996" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793114073-UBiS92dSQTBme4AGdTDE", "proof_sha256": "4d6dfd6d20a431d8b804d1ee3631b5c5154a5108920d083633680dcedbc91405" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793114104-e1YZG9WIXxvB3xoYFCr2", "proof_sha256": "36abd34787649b5a59ecffb6ce249eaa1aa573c8746393f0f0ae66a0e5ba5000" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793114159-6etMT75PKGj3r3HZ5QvT", "proof_sha256": "39b3093fb21bf23d0c37bdf97ca6351ba14007a2080c8d2e7b9ef9cb9e4cd267" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793114229-kquBnL5AQzUAhUvQzMtl", "proof_sha256": "299d5add86e8689847a159d6b59344baab8605301120306fb3ca5b358a62f0e8" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793114310-HjE852HYgFtMJtuq8V9b", "proof_sha256": "09e8a4fc3feb3238a303cea172fad552592d15e9f9de769672ff0bbbc548aded" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793114374-T18LoqLU4GcsUE8uSYHr", "proof_sha256": "5654ea672912ac8b7e47196a688aca5280371aea8e893a4ea6fd55ea852066c5" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793114446-2vv7qj22Lz7uZJ7WIQXa", "proof_sha256": "b9a5f8047e740de2049b8c410fbb54f4aa87660fa79a7445e2537e3d4cf858ee" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793114527-CnjL8vjHqSlHHu6dsvim", "proof_sha256": "e3b631f43790a19f35d86bb1d340b73ff2389f46449b6385f67d2a337f913b64" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793114599-wfN5NsfRvxVCP8DhCPpQ", "proof_sha256": "8ad1dbd2323635e5744ff93a2fa4896abcbbe535d5ea11c7690f83173e6794a9" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793114680-Bz43lAtVtdyAYtBUt2Y7", "proof_sha256": "64a93d707f64756c76454e89f38a1ae06f2474ec03702a2a40544300961238f6" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793114753-eCD7NXevkAKsqLVNcIyd", "proof_sha256": "d1efb8538fce8d36374f6aee44dc7d1c4cb3272c92335f0d82dc55675fc4069c" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793114809-LvLe1sMNgXMmI9kUtlbW", "proof_sha256": "d740472ac53d9a2a42e4f431f2720546e757eaef2e56f2f69611d31b75549c6c" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793114891-HUnl3ayIklrIrqBNfn8b", "proof_sha256": "f87e292bc6907c582b8182677202f0feee5ea321c82013646a092e147307e70d" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793114963-R9JWvhC1wCmffibH3Ta2", "proof_sha256": "cdb3e439212c07b680a29250c7542b0495adacf03c72c75f44c13b1a631ccbad" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793115045-aWcCF8Yv9bish92QzxQQ", "proof_sha256": "ebfc09b7f4ebd0e84ddc7d7b2ea70da96285204c3da533d135122bd2e619b2e4" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793115126-ppohClIylxrNQRrL4Yas", "proof_sha256": "9fb46d1cbba761de1259ff4b9c47770b7db44676081d5b9cea6f6043640854b2" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793115221-TkiDfOLRBaNgZuzWd1vN", "proof_sha256": "5a024e8f8128d13b73a9c4c25f18faa7a220241abfa8423d33aad0359541a7f9" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793115303-TwoZLICe3QtLjcVib8aT", "proof_sha256": "333087cf565cdd5eb1503869b9ae8f7d591aa72320a14d56d0d703ef219673c3" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793115363-2sdUDAe6gbTdKLbORgyo", "proof_sha256": "69c92b195db71acf2c9cca272d10f64b94dbb0297c685de9618c1f93f9a7fc88" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793115423-sahZyc9UueZevlQZ1O6G", "proof_sha256": "54e504c0a0b36b3f9e75ebcfe4b58f0adb7eff23f0667364dd68aa7429bc2c5e" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793115479-Hz9fMBZjbitsnfy0VKqo", "proof_sha256": "94ec578d4ce98da7dcb4169ea4aca49c7b472bd77111ac18a9b30a2115284215" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793115517-oQL5jNqIA8PCd4rvlnKF", "proof_sha256": "0ef4017c05a06388266d3dbded19bc7d1ffb15045a7fa93de867dc7534f80d66" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793115556-YyLVGUjskhd89B2aRINI", "proof_sha256": "9bfb5bebbfcd544bb5bfcc61be245fad88a27cbd6560f107588e05b60c5f39f7" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793115591-tFCvAx7JjH9oErJe0YE5", "proof_sha256": "d0a52744804688f6cc658a3be378fa8837efd5ee7fff456c637e46cb360aa17d" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793115667-0oLw7NXw6h0NfC425P6y", "proof_sha256": "908d95b2710a34050b20a3fc00bc896139a0fbf56e0ba0881fc64beeafa79319" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793115770-tLl7r2VVYictderEPlPL", "proof_sha256": "f98d36ba3aed47ca0ddf0424b09ff8fe5a51729f95f78f0b7a5463cd99e8586e" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793115791-pXr1TfPyRDJNqIbQRjIs", "proof_sha256": "587dc715fbd3ab6572090c960a6bcaf44931d79b38e42e546b1ab3ec6ec25c75" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793115863-ym4JijVhkBrdGQmomDAJ", "proof_sha256": "1803e7be6446b56a9bd92aad3694c80a0148b5117a307230a3f7535ffbf54a8d" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793115917-6TOsRQYernG9aOOMsZWy", "proof_sha256": "53fc96000d7747c8b523cde7483edc19605bd20d47c917c5966b115c143e6496" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793115989-tiefZWLcb8L3qXA3yczs", "proof_sha256": "7f0f4780252c44569044f8a22e20c3777c0b322965978e38d5667b6c954c8cd7" }, { "model": "qwen/qwen3.7-plus", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793116070-MJyLxnUwTJJP0vpBw0kk", "proof_sha256": "82a35b1f36c96e0bc61eded036ce66b7f3be48721045b77d82bc7073f71c2c12" }, { "model": "qwen/qwen3.7-plus", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793116097-mC5M53hBpAr3eMd7v6hP", "proof_sha256": "7ceefdd2e1e63aee2197d854285beacb14414fb68e88ce7fddd05e5889b73b11" }, { "model": "qwen/qwen3.7-plus", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793116128-Ki9eIQ18iNH3Hkmwf48Y", "proof_sha256": "5443b8a81d7d3e1ef719134973ef2aaf8751eb4fb9aaf8e5b276b8f7962cc93a" }, { "model": "qwen/qwen3.7-plus", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793116189-uCBnLCAnOKjOUXPl1ygm", "proof_sha256": "f5a17cf4f884c8438ab98ca221d097a1bbb809f6e749875e521abaedba7cbeb7" }, { "model": "qwen/qwen3.7-plus", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793116244-O0vFPbG4FbdB9q6U7hZj", "proof_sha256": "583dc377ad54fa0ae2d561a073b71326a3785bed5e908b357df949f1a3b54315" }, { "model": "qwen/qwen3.7-plus", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793116330-KYQa2pcUlqyU08HDwXEl", "proof_sha256": "aa0119935ea237eee7d652bd41e5714ab389762942129f69291d77de60b85d5a" }, { "model": "qwen/qwen3.7-plus", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793116371-s7PAYzyxXBPwtP09ePo8", "proof_sha256": "e7619c76172da0c3a8152c706da4ef2744213607f3c7e8a5b42652b8fdf34a34" }, { "model": "qwen/qwen3.7-plus", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793116389-CRJvSFWsQxCkeSS6xRjt", "proof_sha256": "998e03e0a0fcff38f03ff0ae177c34894486a2efae78726cc0f0ed1f40347c86" }, { "model": "qwen/qwen3.7-plus", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793116413-C5NudW0aDjn26zxFDacJ", "proof_sha256": "8ebd1523327ea483cfd89e878ac7e3ac46f1736d95f45a966b3ac9d4612aae7d" }, { "model": "qwen/qwen3.7-plus", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793116430-X4mp4T3Pw8JHGosJ5jIU", "proof_sha256": "b80b285c9a2112eb0baacc2892843b6de0bc281bf53c7d1ebe425384d1e64bed" }, { "model": "qwen/qwen3.7-plus", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793116452-GQMPGBv28KneKclc7VuS", "proof_sha256": "0a68337c2947c07d6d74cf14188d060b4d3a0c4ad5a1d01995eee43e3610af8a" }, { "model": "qwen/qwen3.7-plus", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793116476-eXuU9GIDdG4ZkDrPoTq5", "proof_sha256": "29264e2df3ca9729533f2f6db4fb91f510f2ccf1d3c498b1335e867c591e6130" }, { "model": "qwen/qwen3.7-plus", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793116496-0MJoo51st0SRarlPizGW", "proof_sha256": "16058d84e4ba8133d876ab77def220ec649b7f806589be1d60326c25e685c492" }, { "model": "qwen/qwen3.7-plus", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793116511-LGRucMvO9UJdsj4hkRgt", "proof_sha256": "f55d2c21ed5ef57657402ba0558b51f97403d4bbb61aa99a55c745fa7bbf0295" }, { "model": "qwen/qwen3.7-plus", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793116564-l5sWjBht56yTNtV7UDIb", "proof_sha256": "5fe420e9d9dc62dfacba0d126ab833f621827d97f028d4d68560dab35961a0ee" }, { "model": "qwen/qwen3.7-plus", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793116584-ccBXjSuLEc5RAkUdSKV8", "proof_sha256": "a069d50fda326595b7ef61e8f376e4b35d5568750a674eaba092036bda48db77" }, { "model": "qwen/qwen3.7-plus", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793116656-P5m49ptEOz3vRgD5SzOE", "proof_sha256": "5a3a693d7d536d6fd4c8a8dcb433d49d914292e2950c147c222389e041b9753b" }, { "model": "qwen/qwen3.7-plus", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793116672-kdnPjapWJoa81wvDHD7m", "proof_sha256": "5963f6793b0aa2fe85c28b4a2f40d9cdb3a9ce895e3a423e2086c1147ebfd351" }, { "model": "qwen/qwen3.7-plus", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793116704-LiGkrrITgE5SixA6YYsO", "proof_sha256": "f61e13cef9fb22f0a704327721362e3cc2a3df8717327f6ff3c43ff15da3425a" }, { "model": "qwen/qwen3.7-plus", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793116757-u2M0oEEIsHLdJi3dufO0", "proof_sha256": "1649b76fd8aab34bf7bb964355ee73972d1b9c5946919c3f9aff4ef89d1ddcdc" }, { "model": "qwen/qwen3.7-plus", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793116773-D141xAYDKdukBZaBM8o1", "proof_sha256": "3c632c50672d51cb2435a6fa27c6636c8e3f4a5dfa06889cfaae7ea76af7b300" }, { "model": "qwen/qwen3.7-plus", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793116802-hqYCgHuHTSPnrFinDIOi", "proof_sha256": "4272652a675312cbb15b88cae47a6ebe378093dc320ed92480b5df30c998ebef" }, { "model": "qwen/qwen3.7-plus", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793116833-L0Kt3g9K8djpHipMEaY3", "proof_sha256": "a566737148d143d9847087346cf826ff3d8d9b46cc4dbf23dc2298506b88afe8" }, { "model": "qwen/qwen3.7-plus", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793116885-F3r3JtjbErtNp0IVT8su", "proof_sha256": "4ddf23cd58c1614ff2089c275068c97d17b9f91af844c681ef6b744b3fa54b13" }, { "model": "qwen/qwen3.7-plus", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793116901-O8RebTVbuRnTSiFioy8d", "proof_sha256": "4f6e0658d67e3013e7bbc5c272a48784c63e00f5d3d36b67bffb742d17a86152" }, { "model": "qwen/qwen3.7-plus", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793116939-LoCJr3qYCFOFvJWePl2Y", "proof_sha256": "7e32379c120c962a09fe3ad4717c72739cdf80dc47ccf62831dac5dcef145e98" }, { "model": "qwen/qwen3.7-plus", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793116976-jGSeAcNS9yPlpQxsH9Jk", "proof_sha256": "076379b0590333566fe10010e0d0a241c257a1e7cef3daab7fff6f8ba4206a7c" }, { "model": "qwen/qwen3.7-plus", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793117008-jf5tJnpP5Pk1xlZPWpwM", "proof_sha256": "af649a0539a8a4b1a8710728aae5cd4b8f12d0a77353a303e35af33ddfef9fd7" }, { "model": "qwen/qwen3.7-plus", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793117081-SZWH2H65roc4M71w63US", "proof_sha256": "4a0ff4b481e529e590b1d8101816f08d0be99a9464eb83c269f63b8f0d2f118b" }, { "model": "qwen/qwen3.7-plus", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793117100-APRgy90CbYOQuieDfgGn", "proof_sha256": "188b05ab1719b029a52b076a15cdfc21c855b21e8acd1a29694a3a7426588966" }, { "model": "qwen/qwen3.7-plus", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793117113-6WaZyZm1Co4wiBwiGeTq", "proof_sha256": "c22e0cbdc424e88202eac74f7abb76ecfd67c23b8b057a0ecdfd03b4c82d169d" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Crusoe", "run_id": "20261027T120517Z", "generation_id": "gen-1793102717-IXGauGJAMnGx4RR7mtde", "proof_sha256": "7d0212fc26d99a3701943705d43c3079e56a503db02f5e3cdf707b6b753d0ad1" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793102751-Qz0qGAlPVAwctvBBvK29", "proof_sha256": "e1fb5845aee69c4eb1f71b843114b748d3532d92d04ec22a73835cff6386ce20" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793102960-lnNSsOEWXwPZ4W9WtnWi", "proof_sha256": "42b997c72d94bf02c6fd68687f6edcaca305b981e6428b40d4799c52f0b5553d" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793103002-2FZCdGpMMk2Rw82gZnQU", "proof_sha256": "558750783391222d98f6b0585fcbcb75c1cb44b48b61bf64f3a1156c69cd3334" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793103047-L7ViHTGPWilqWY7QSlps", "proof_sha256": "3e850e1179a09f227553c862fa6c8084bd9309cea5c5fde94d4d95cd5f78e43e" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793103104-d0kDplC5whg9nQlTVTik", "proof_sha256": "01c2a0f3b5d5db0d8cc83d436c2143bd381c106cc48e12358a106c825bcfe8d3" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793103268-IwN8kScDbb92dIJe4Aev", "proof_sha256": "3e15b4f7548e06480aa755231769bdbe8f944a8b04da30e7a2f37109b8571878" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Crusoe", "run_id": "20261027T120517Z", "generation_id": "gen-1793103325-4pzk48M0EdawNzF3nZsF", "proof_sha256": "99f26ffcd08d8069b27c60228fbb3df90fe503de2ba76493712c92a372a19223" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Crusoe", "run_id": "20261027T120517Z", "generation_id": "gen-1793103344-ix3D4Oyd5icZI5YyErs8", "proof_sha256": "09d736f3fdf33c06164c2bd4471378e69cfb00212e5fdb82704c0503d5e59ce6" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793103378-P5j0OfI76aGs6t5l7U1s", "proof_sha256": "eda43d41569a5edda367d343a20f2c0cd0dbcacfad70c81521e751fc4973f887" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Crusoe", "run_id": "20261027T120517Z", "generation_id": "gen-1793103410-6SB3mOhZdF2CzjLmX1sa", "proof_sha256": "54b5faee653eb61caa3724cea8915278a7e98505a9cf1469309fde9a945d0e9f" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Crusoe", "run_id": "20261027T120517Z", "generation_id": "gen-1793103432-VHY51I7jOvYZOHpF43LL", "proof_sha256": "d419956b00f28d518c9302544a2b6df97df955ed4350cda98694bf31c48bf3ff" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Crusoe", "run_id": "20261027T120517Z", "generation_id": "gen-1793103466-zsSGTngXXoYwTJSjZe2u", "proof_sha256": "ffc7b703c3b6a6267d33a2713a1c885e2cdcb413019783be2e4070c0a8a18a4a" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793103475-l9p8xWLPjp92oDuKwxB1", "proof_sha256": "6f7f50dba19f95eb7a7223d3494f577b64c15caea11a7821c7fc0406e9852930" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Io Net", "run_id": "20261027T120517Z", "generation_id": "gen-1793103493-0PdoaUJO4Vg2BmoAXmfG", "proof_sha256": "e9f7323d2394d7e4542150991eed845b1a40c3142f0ca15083fdbd7721aa88a5" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793103681-FkoAdRnk7lM2gvlzHvmp", "proof_sha256": "4724da50b9ee067976e1a907099f99a2d9a9cfdc62df42ca5bdb61f60177e302" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793103725-vXtadfBEBCXEWTaocLj2", "proof_sha256": "4e80661ef6de6cc905b3722bc880768c06c48913a3b30e8ad9cb72eb85e8cfb9" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "=(1.091\\times10^{-5})^2", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "BaseTen", "run_id": "20261027T120517Z", "generation_id": "gen-1793103787-E82nUcFalOzrN1bJmmBy", "proof_sha256": "da0e8449ba616923ab3b7c4403cbd9c2fff0dc90cb1a44e583e48b405cdbcebc" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793103811-kIrsx3GaT5Y8BvtSTzUq", "proof_sha256": "319acd7f3370f1678fed79e7528d8cc17f2af9d3bfa256cb20db11e4ec0b614d" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Crusoe", "run_id": "20261027T120517Z", "generation_id": "gen-1793103850-uiYTLkmrEPV2gVodlTQa", "proof_sha256": "232bdb078581938237008c7b7ad8e57bd815634f745a2f06aa9267448628476c" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793104325-1U9CCkVqoYTSsJaXltLy", "proof_sha256": "52a1d141da9e0bcc1c83e59f29e138ae61062bc2bb924b79737e320afb34bab1" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793110115-V8Z33s7TxnURULAnklOY", "proof_sha256": "6d349ccf1613f84c9f527b3f16b3bd22beb2a7ebeb62773e0c834b13f1d4a9cc" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793111031-pxusNC22ckMrU3hnenkD", "proof_sha256": "ce895a51bfa1477c2937214e312b1da1406da9763d4bab9613059f44fbd4fdc3" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793111956-mHkAXwjh2YfDFx80ZXXH", "proof_sha256": "aeb3973d783c93f32da7c90ffc35cf1f3ea10e6dfbf518538180112b18331203" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793114018-vz0M3IzeeyVV92AIeSBy", "proof_sha256": "e88d9e5487f84da649284a935a77fe3bd0e494d838d2752b47072c0522e4188b" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793114161-XZpO41gSSVkcvhee0SYs", "proof_sha256": "021a5f11b2311172ad5eef37d56b4661a538b5b6f2803d0c901867d4185ae47b" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793114201-Qb9utOzS9Y6tQU8wTvuE", "proof_sha256": "14efa780a7918e1d12786a74f343a7a0b8265f50522fa2da9261f564c8132b11" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "BaseTen", "run_id": "20261027T120517Z", "generation_id": "gen-1793114355-O0pUFeM04RwytCO8GzPd", "proof_sha256": "58ab7365074dea307ce6409536ec025463a095e352c67c7ef0ee5adfdbef3bb6" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Crusoe", "run_id": "20261027T120517Z", "generation_id": "gen-1793114376-1OlSt7DZxErQcjJ716pR", "proof_sha256": "1cb421030621a0ba59f44d39fb12889593b77326765c3e961a698caa6c1cbd36" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793114404-Q36KsiJZ9n72C2C5rlbi", "proof_sha256": "560c03e2686e6cca53c5cae8ad3bddb282a8fb08b818de09b41d5884b81eb88a" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793114447-aTaBPG0sqoqkV8r1UxdC", "proof_sha256": "4fa0c716a95df0d525f11a1677ff94f0e81641ce6034be42c9298a3039a15f34" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Crusoe", "run_id": "20261027T120517Z", "generation_id": "gen-1793114463-suZuE412X8LaTKdMHCA3", "proof_sha256": "bcda75c3da26f924f5109c475f473ce20648da79bf5a2d566f081c04cbb2f2dc" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793114499-WX5gB0Ml59H3yzIKtLSL", "proof_sha256": "334b5b0eeea9f1a420a1b3ed51fad74664aaf15a069033396d2eab5ad209247d" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793114523-aSCvHgjrNl0ynW4j8dPl", "proof_sha256": "575074ba04e38e0b304078aee08ddf1ce2abc1255b8ad2f309e7e03de3227434" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793114698-oANSe6hO6idmEGwjq2Vp", "proof_sha256": "644502753c81ef2a5395f403c67b0dc13d3852d67013584e078df30a9547746b" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793114860-8i9SIMYtsP3g5V1CLSts", "proof_sha256": "f21e0cfe5c31250f2b11471af5e98ca38e95921382d04d3a2ca185a7d7134158" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Crusoe", "run_id": "20261027T120517Z", "generation_id": "gen-1793115045-6XXix1uxHIFTKuFxljJ7", "proof_sha256": "00633e707a113811020f90452b3a3a93b1a53ae0f1f992a7becf39cce90f0560" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "BaseTen", "run_id": "20261027T120517Z", "generation_id": "gen-1793115076-g2mSB8raR1oAN7HYxxni", "proof_sha256": "48f11535da6197989da8c8c9f5b04324c68f6a4b483412e2a47951a8550737a0" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793115099-mRkTZmC0kn90t1qlG9Pw", "proof_sha256": "5eb7fd8bf0bf1cf2e039f020114916f22f1e7bc449109991b3c88e4ab245a4ad" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Crusoe", "run_id": "20261027T120517Z", "generation_id": "gen-1793115289-s1EkhLv2ZOUG4WRe4R7Y", "proof_sha256": "124b15db5a52cf70817b15722db45b7b5951591971ff3a72d5c6534bd8c90b14" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793115321-QfAqtVCswE1voL3iebUV", "proof_sha256": "e8b8c6e5423d472694556d3cee603585cb823252ffc5bb0cc9a078abd31a4fd2" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793115509-qhyUncB1DGTzlLQlF18u", "proof_sha256": "b60f2cffeb60a2d438e910023523781a17ede319848248b57c5b6aaca7c13804" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Crusoe", "run_id": "20261027T120517Z", "generation_id": "gen-1793115524-kS3vYvIEIMPbbWTf5Nbo", "proof_sha256": "ec5ad8b495b5148814308051d039504eafaf2f98ed68e6aed3a77f1ea8baa632" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793115552-dVcVFQMB2mBegUTHtYt8", "proof_sha256": "e0e698c55e8a313863618cb6c9811bfa14ab971f29883307464410e351b7885f" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793115764-GufzarJonuJ3hbr61zyp", "proof_sha256": "e39799b6b63c6ca3228ced0eda69c836780cc865eee17b13a2f81a92ee4bf7c1" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Crusoe", "run_id": "20261027T120517Z", "generation_id": "gen-1793115799-NLQPIRSodypxzT1Yl7Xy", "proof_sha256": "4c9829e015883bf4f41706412930eb77b977b2fdca981ea32e7340c8b0fc8ab4" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793115828-Iu4SZxwpHuNRUWKBMJii", "proof_sha256": "6bd65ecfa97902d4df0e74b5d899cf89692bc96faffbea73509d79b5c2ffe227" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Crusoe", "run_id": "20261027T120517Z", "generation_id": "gen-1793115863-pmkRDl9Hr5mfSBUWpBHA", "proof_sha256": "d51633902e17ca685cfe444482c0597dfffa41d1470730c07e961eac9d09696c" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Crusoe", "run_id": "20261027T120517Z", "generation_id": "gen-1793115897-zJLqlniW077Ca7Zl91k6", "proof_sha256": "dcd57340b92d0430ef2a0301cc9123e59ba98aad6be30bd09ba0d8d4df5fff1f" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Crusoe", "run_id": "20261027T120517Z", "generation_id": "gen-1793115947-SICxPjHQZMp9jRYFiKI8", "proof_sha256": "6cb87fe17251dfdb99dd28afe6f61ad2dc88acba06141544bc6963212d827e8e" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "BaseTen", "run_id": "20261027T120517Z", "generation_id": "gen-1793116003-aKclXKAa91vHQK8AYXEt", "proof_sha256": "bda4ee7af0eafefd224050a4228e2171f3f9b43f5c4bea6828e3989aa2fd7b6d" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793116028-qtcSn6pozoz75IdqmUBv", "proof_sha256": "f941743f909170946842fd61a26249edf318f448aa0d1fd104021c9b17e47737" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793116167-A7L4JoI6VFvPDFxTZ1Rz", "proof_sha256": "7b50401005a57cac3fff45b858b05d296b8e228c623a607233d64436dd11b618" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793116181-OLeOoavtgfsfcUqY8fzu", "proof_sha256": "656720a679e376a557a4ca9bbd38006a7e0fe85a4b29b921fd7b70c7936f6a52" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793116229-k9BiZq5NcU3ZrJCsFuyD", "proof_sha256": "a6cf3259229ac0f6849c7031b4fe24ed2e613d1e16ecb25b6b332e43b2a0d379" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793116411-XBuumWtOSo23QUosYl3y", "proof_sha256": "589d6a27e6efcee77d6a4187d6babec477fd4651e80eb3cecb07c9cde01243f0" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "BaseTen", "run_id": "20261027T120517Z", "generation_id": "gen-1793116595-4pM0XbhoZgB6tsqKGFzO", "proof_sha256": "017ac6855214e70760028b6c7e378ee74e280ac1c4248f777770e6e1df1566fe" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793116605-oyPbHs7aFdcCg5Aux3dN", "proof_sha256": "c818a6325e2f5223ce6369267eb31a394c34d9a9e1669e8dff1ad875f3b63066" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Io Net", "run_id": "20261027T120517Z", "generation_id": "gen-1793116622-caltpEYewEE5bTyytWOT", "proof_sha256": "ca907b152c62558e0ae325b997eb77c8952866793683715634998ec549194804" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793116638-PFbsiSBQ7TSzmU5wja6C", "proof_sha256": "e72068062beb8e4b2db6964c99b6034240da72cb245f9a7f20d044ac7e51e467" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Io Net", "run_id": "20261027T120517Z", "generation_id": "gen-1793116669-ZDLJxw3GTgBLtBU7LBB3", "proof_sha256": "87564f85daacfbd6cc6a162dbdffeb785c6087cc32081ed3792d0e72541e4e31" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793116691-hJ3mA9lA41IjDdLN0l2K", "proof_sha256": "aa3d22812684afb93de46da4a1a4f9a3d395f779c0ef8498074eab3e41b43214" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793116707-EklTlMqTyuNQPbrE07Sw", "proof_sha256": "4686a648c4d21bcec375682d5bd406d532ba8c84e90908411826b45ff5081be5" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793116742-iFv42l9PJxR0YWt0idVc", "proof_sha256": "f54db9e486f8ae8eea9986958978d89dc88ea80fd72a11a848e677ab2f4737c5" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Io Net", "run_id": "20261027T120517Z", "generation_id": "gen-1793116766-GZeaDnxuSiLDfNJH3bJf", "proof_sha256": "8e1b3072c58e0197c8f6dc7596e89fe5736e9535ec5d8da72271b54e93a9ec52" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Io Net", "run_id": "20261027T120517Z", "generation_id": "gen-1793116852-iCLZpLmW8q5nTko8YFV8", "proof_sha256": "c8c873c1c0f2d270751fcb0e5f08dd287071b9c8a1144c6505395bedb570461e" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Io Net", "run_id": "20261027T120517Z", "generation_id": "gen-1793116872-0EYmzwB5UvJvWKzYvpF0", "proof_sha256": "ae5ff3636f72c25f357a965690ec16e8034946cbecaff54f919f0ff694291a09" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793116887-7RjLu8dgILX7lTG4cs68", "proof_sha256": "f835c89d7313b51c411e040faa39ab2266ce1981d581bd6d57d20f65268f7c53" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793117056-DlnqAFudgUbXgXaLxxxP", "proof_sha256": "9bfaf35bccd75900c3b064be76be58e11750dbec2337f64769008533aafbacd8" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793117084-m9dxFUqAiCFy7Z57dcSL", "proof_sha256": "2737ea46f4663ae1d2fd3c3ed861e318721103a1391bba12d9b16781edc71263" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "BaseTen", "run_id": "20261027T120517Z", "generation_id": "gen-1793117121-gVRYuPPpTctn2Y22vlKe", "proof_sha256": "aede4d51d4a21f1275ef551f79b801faf4bb9341cf257468373b1b7b74bd7b46" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Crusoe", "run_id": "20261027T120517Z", "generation_id": "gen-1793117138-k01UA2mdUK1sDosn2D08", "proof_sha256": "990a5b0a0635aa487c9c693759fd5d6ca384e1c61483794194377fa5a6491148" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Io Net", "run_id": "20261027T120517Z", "generation_id": "gen-1793117178-tbc5UagAEI5xm5E9p7oT", "proof_sha256": "d530f101b3ffbb556b70b1a2622e07aaca486460ae3b470f737af5b1b9e946c6" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793117190-E9hu0FN8q6jnjB2Xy0SA", "proof_sha256": "4810642297d5eab3f831211f1ed52c18cc6cc848c17675cbf5e5476a7e9fc856" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Crusoe", "run_id": "20261027T120517Z", "generation_id": "gen-1793117210-fMY20L2Bw0OUIwT9S16T", "proof_sha256": "3281336d1c034f0004962dcaa5b5e2de9247386f09bc91dea4a5c945e394fb8d" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261027T120517Z", "generation_id": "gen-1793117227-QDIbIEd82l9Zs3xrI4Pd", "proof_sha256": "750db1d8f14a98426d691cb5593cf3613e9335c9cc9895009a0274ce9f3184c0" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793117246-1qlKIgOAmPbPXl3y1wfT", "proof_sha256": "9a1727b4b775dc5c8a65161bce4227220716fbf4940d07680923d89a6eae2219" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793117417-Dm7bXNJbXLMdd1q3DELA", "proof_sha256": "84dc545bc42ef22f28aeb94c74e614b4a0987ef5694aa5f22cd2eb3385057c19" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793117446-fHm52bQy8B3bwDCIWP8H", "proof_sha256": "ce88462321e23762fc339b31bdf2e37ab19132fcf09e1b49fa8e3345ceaa23f7" }, { "model": "nvidia/nemotron-3.5-super-vl", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793117472-ZPFfhCAtxcKtbsNgrlb3", "proof_sha256": "cf01f80b608c38a526707078160be86db76dc8de31080e3c04577f02c086185d" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793102717-bmgMvrTCxrrkPgIEz18Q", "proof_sha256": "5dd9cf382b2d8a223313d332d82daddafb6e0aaa7e2c098e0a412dcd26751526" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261027T120517Z", "generation_id": "gen-1793102763-jLFMGJvW43Uw3NfpM4hh", "proof_sha256": "e1f1889b5a2476ad58d1aba4e1c2829ac1fc80794b928d35c7d9f430c3679a7f" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793102795-IGQBDuNlJ9EfWCI4H9yM", "proof_sha256": "cbf7015057a455de6a1274d3f19c5dd3d07b0d40953f2690e555d526e13d2e78" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "SiliconFlow", "run_id": "20261027T120517Z", "generation_id": "gen-1793102883-g6ZGm7cSTYdFdCUOrOul", "proof_sha256": "06aef269874ffab6190a3f201f1ae21588eac0ee3e20edc15571f505b3742cd9" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Cloudflare", "run_id": "20261027T120517Z", "generation_id": "gen-1793102945-uV6LfSyypuptgXxabWPD", "proof_sha256": "e9319ab77ebe57f13359b40e60309a2baacb673684a1fd19d941c2a71102d893" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793103064-B80HbvXTtd9lKPTkEfKh", "proof_sha256": "56feacf113741db5282db30dcce69353cc0baa04b0808ddff4ff418f9827a1b4" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793103137-H562Lqv6yv4IWCMemao3", "proof_sha256": "8184fb1e5b23033f36a9d38c8b0b2e3c87e463b8cd79a81f75047d8a7b9c8053" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Venice", "run_id": "20261027T120517Z", "generation_id": "gen-1793103231-w84P4j6Jh7djLj14nD8c", "proof_sha256": "6307d6e09b1f91376353cd725a13cd68c15ab5d7d74fdde08bb64a845de12df4" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793103285-XGViklBKy2Oj5xQA5hN4", "proof_sha256": "161007af1731160c818a1459714aa7692c39bbae0b1946505808e535f696e0cd" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Cloudflare", "run_id": "20261027T120517Z", "generation_id": "gen-1793103380-J0d3jtycV7qDNrFtb56T", "proof_sha256": "98219c4b4000fe8a1f8f869d3d913605b8282e1fa909730016661de2f223cfaa" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793103476-5o7Tacg6kaAevWrlvnZO", "proof_sha256": "5a605ae2b299ea87baf0c2533959bc84a815e9fe079df35eed20113e5414cd1b" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "PrimeIntellect", "run_id": "20261027T120517Z", "generation_id": "gen-1793103517-G9tN2VkRnic7uk9e3jhC", "proof_sha256": "2cfda3e41b666778a6365d5a3be31dc18a09607f139c2bc28ab5dcee0a910d97" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AkashML", "run_id": "20261027T120517Z", "generation_id": "gen-1793103542-FTFLOVEEYfXdHYJkZC1I", "proof_sha256": "293d02a3e5455ccbb59fd336268e16218c3c174c12db199ef8ae677a4f380b72" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793103582-UrgO80yiOjAJyorroQIa", "proof_sha256": "6d6721a6ff9c2fb8aab13834048a357fc434f39906d23e885909d9185fbd6294" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793103658-RfdrAHuNcEe2LpM1d87q", "proof_sha256": "8e97349274d0db3d80809bbe11a2a00b39ea90356c03c8a860e5a5cc38ff7a0e" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261027T120517Z", "generation_id": "gen-1793103726-tJ5yhqmEy4VZLPxBFZYt", "proof_sha256": "78ba944c5f67c7e9cb1484a17ba3a04f12b6d6f5cf03473ba72c8b22370ef2af" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793103748-LPxC36zZUWlI1AnfhaFS", "proof_sha256": "e13a3addd5a290252f4bd712a14ac4a8d9ccb10751564db2796542efb80da06c" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AkashML", "run_id": "20261027T120517Z", "generation_id": "gen-1793103802-mmknfjd3WX2HwomgUV3Q", "proof_sha256": "c8aef5988acbb5f87c381b804a33ddc065ea0cf3a171659765aa1e94f37802cc" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": true, "finish_reason": "length", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793122291-2ZPPjQpNvaKLbF1Yw45P", "proof_sha256": "9b50d23b1bc6513b66d5a3d796125f8689326ab94e112b4f10cc31bbf6834b1d" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "NextBit", "run_id": "20261027T120517Z", "generation_id": "gen-1793113931-X3EF0lcl7twsuxYM8RTT", "proof_sha256": "384a47b1ce314f72fc92eed333b65cd77131460c12268faf6423a90d321b8487" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Makora", "run_id": "20261027T120517Z", "generation_id": "gen-1793113991-3BXwkguy1Q55912BT4gc", "proof_sha256": "7566a56722fbcc90bd710d0384c7157a45e6f014ddc5133bed80c4c679bde2cb" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AkashML", "run_id": "20261027T120517Z", "generation_id": "gen-1793114012-0T4pTaPVZHxhoaMlsaET", "proof_sha256": "09720af16933e6da1810cfefaafae0b9b8391d1387a5267e1b3f3e05563a0b65" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793114221-DRgJ67PX92lH04pOQY91", "proof_sha256": "318f6ff810ccb4bafbf4c208d1a6b76aec03c64a063413d61dbb4fa9cb1da170" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "PrimeIntellect", "run_id": "20261027T120517Z", "generation_id": "gen-1793114270-m9LRMXxvSh6WtY8ndR1f", "proof_sha256": "213f6d0c6a352f289c8980049abb16629ae586d63b7e331d814b28cdf9f5fcd6" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AkashML", "run_id": "20261027T120517Z", "generation_id": "gen-1793114319-UGvoU1cjGLf50MwHVPdM", "proof_sha256": "537ab6636e7c0adecdc31db790324b9ebe7881e995ed5ec8bb100c361fe81133" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793114614-i9Caci8UDRSE3fev0NhZ", "proof_sha256": "530e8cce1ad5ba2c6377cfa7e14c8d0eeda18ff831981867659e58461225c195" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DekaLLM", "run_id": "20261027T120517Z", "generation_id": "gen-1793114689-XSRrxXiYgLdIIKmS8Ym4", "proof_sha256": "0fdbe6d7b8edd5c77261e7c3b6d6151662e87b0cb6458b0d3af3d6ed1431855f" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "PrimeIntellect", "run_id": "20261027T120517Z", "generation_id": "gen-1793114761-tWlkjDAMIM50dXN03vGi", "proof_sha256": "178f9fdec185859d897707abe579c8daff769c701fa9fdd96d3de5b656b73f83" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Makora", "run_id": "20261027T120517Z", "generation_id": "gen-1793114790-D00IloFLwfB66uVbrDPk", "proof_sha256": "f620f63b6d1341386a2419f9054f65ff054b1b74934c10cd7328fb2372659ab6" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Makora", "run_id": "20261027T120517Z", "generation_id": "gen-1793114813-aczQRkzu28HODk3MJBxI", "proof_sha256": "0f2a3dc0f0845c681f7a96b03fccd3c088499e68c19f2bea16c9880caf0e740a" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Io Net", "run_id": "20261027T120517Z", "generation_id": "gen-1793114837-OEOO7YIWsgedci87OmgY", "proof_sha256": "3a061e157e466e696b12afbdef23592aa224900c6cd84ea0b8a2362d0aefdbd9" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Cloudflare", "run_id": "20261027T120517Z", "generation_id": "gen-1793114885-1XXFNsBr1Y5GP1mOI2UW", "proof_sha256": "f89f4e600075f287b25c9875cbac9729468c09d176c5e6e346f22415c9d37a9c" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Makora", "run_id": "20261027T120517Z", "generation_id": "gen-1793114984-Tl6SUP8KVyU7B0iEj7Ox", "proof_sha256": "26f30f1c6235ba2983b30293209d0f8add1840128db686d34ea6dcfb92797633" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793115010-tcFBP4iWrM0P9bGGRb4P", "proof_sha256": "7f2a2eab5be8d442aa3e3b4c99aec06fb343151a04c8ee5d729cfe592cd9b7a5" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Io Net", "run_id": "20261027T120517Z", "generation_id": "gen-1793115066-oUzUzt30OzV1UyXZ4oRY", "proof_sha256": "aaa9b5a026e41cbbd2cc3e043aae626faecac89ed6cb773032a174d8d5eb5e09" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Makora", "run_id": "20261027T120517Z", "generation_id": "gen-1793115136-PpX5btOraKZmuiD8OTUj", "proof_sha256": "0d196760f9cb1f823330ee6cf492613759bcc3d367a7427dfe4f1e70a606291a" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": true, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793122351-mdE9Yam653bfFhQLS8VB", "proof_sha256": "ab3941cd380ecbd3b5cfb4972e544b6fcb17bb8b1e3a44b955ba4746123ab48c" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793115219-XYmQy1sp7dVKeINDU9Hr", "proof_sha256": "5dbb68438fe10c4e5c6d6e62d64ff79d58babab340f153a262b3e6f1c8a48c58" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793115272-u7QYvKiIxtBYp0UABHlE", "proof_sha256": "b8dc60ca510b76dbc267e396b1cdc47e67163131a93c6eb3da7fd56267c1baa7" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793115346-UkyPzDed8HSkTHxVrZac", "proof_sha256": "ba23bd2c70e68f555699afe892f49630192716d93376b36456c580e9a6eeaec8" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AkashML", "run_id": "20261027T120517Z", "generation_id": "gen-1793115415-fDiMjzyVdtxgJRAUltug", "proof_sha256": "2fa505c4eb4ff9e2cb4808df46f7ef0ae8ef1bbdc2f62416bf7ae7fa32264350" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793115675-gUXSNKdIJnnJamGrqoYJ", "proof_sha256": "fb758ccfdff8696f596ffd2496529676ca4083a96d4f3707ae7f536abdfe2ff0" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Makora", "run_id": "20261027T120517Z", "generation_id": "gen-1793115740-r2QIcGcjYpnEUn1raMgb", "proof_sha256": "1422d77e15e3fd6a05cca741ffacec5e4bd0b888ef070b65a2be66a21cb40712" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "SiliconFlow", "run_id": "20261027T120517Z", "generation_id": "gen-1793115767-fsoOGCEir8F2OFJ5yL8E", "proof_sha256": "7736a4206ca5218d53ad79cedc631e2b6020faaf6efb492492936523c46b6272" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793115985-u2r9M0RqI4D4JDwnSBu7", "proof_sha256": "8ae8d44e14a805ed0455489b1adf189cc24752a24754b29e23762da5096a525a" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261027T120517Z", "generation_id": "gen-1793116028-eev9C9Pwcifn3T3hD0bx", "proof_sha256": "0e0f42ed0d29eca9649b0666da0ceaa4fd23b89429d276ae46f90c390a7d5e91" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "PrimeIntellect", "run_id": "20261027T120517Z", "generation_id": "gen-1793116059-Y7xLBLdB6wHN8ZKJSSdE", "proof_sha256": "082f4dd492f166ccf8804e0f8cf1a5d23bfc9457b056a01e91e90d7b795cf4a8" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Makora", "run_id": "20261027T120517Z", "generation_id": "gen-1793116091-7wUh1BrZ5AAxEzhr4ryH", "proof_sha256": "e0d828f55de5328408519048d2ed30264860aa9eb9ac8a8392c6701aa3b950bb" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793116160-DS7HdJHd1G08LQPvEGBg", "proof_sha256": "cd6fb5bd179d9d7cbdb0c0a2c98382d1e9f2e9d8043876a080beac47253640d0" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DekaLLM", "run_id": "20261027T120517Z", "generation_id": "gen-1793116250-P4WLKSFH2J7OqnLGwDg0", "proof_sha256": "db36175e07d915c250c5cd5bd9ab681f734c96a5c86e5c2736aa936174086ba0" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793116348-6nCMkrKDilwtI24GwtsH", "proof_sha256": "3d1d3a3d1d60f67264b7ccfd1d27481533652df6cf74a88127284bb9cf3a8ab6" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "SiliconFlow", "run_id": "20261027T120517Z", "generation_id": "gen-1793116393-2YSMm7ohVgcH70NSrl5J", "proof_sha256": "2a9b2cfa850af0ed6985e265929347dea14d3db367b17637e9d42b582c3756b2" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793116584-5AVRfvebu88rPQYe6zSS", "proof_sha256": "093bbbf7868dc23be3cdbaa8683ea305e5334714f45a63b5071b2838db343ddf" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Cloudflare", "run_id": "20261027T120517Z", "generation_id": "gen-1793116633-u9bpJShwHWRQ3Z6rLvLD", "proof_sha256": "b9349a150d6e76fae9910cedd30b71fcbf805f8e0be5ea842f8357d6edd02581" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "PrimeIntellect", "run_id": "20261027T120517Z", "generation_id": "gen-1793116691-Xb4KOIaobl59NUZQ9W6A", "proof_sha256": "1db0afddc56c25963eb87ef8b600be011bcc4232fecf55f14443937124bc5e34" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793116788-tRy9HV2ADh1zNai2qSKs", "proof_sha256": "6c3df5b354810c26ffbbdc2e802e37c2803bf81bbd69df315fa16d1a716bed7b" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261027T120517Z", "generation_id": "gen-1793116848-Xk6B4JwOMtc0RbD19MoJ", "proof_sha256": "9731dd1718d727538ef11e0e082761e0bc6afda78d8e3e9523f6e286e5b8dd2b" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793116877-Urb7UGXzc0bzVWKGKvD7", "proof_sha256": "b99d257d548a9bdc35a16ead7ce6a0e1960a2a6d3efb6ad01ba1b7269b9e99c0" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Cloudflare", "run_id": "20261027T120517Z", "generation_id": "gen-1793116922-LppJot83aALW3CF3AsiT", "proof_sha256": "e860a2fb068a70950cc91ddb40a9833b2c3d939a128668324b0e1b5ea4f8b22a" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793117005-sYgXV7Ds3QUjn3Wpdkmp", "proof_sha256": "fa167bac6f7f9c8b10414da56f48088156b8ed1e0793852dbd5e8167617d6924" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793117031-YlYwnBMUFKI9Ar4Aum5H", "proof_sha256": "2864a27dfcb747626b98a918f2d8e55975d95027d310c3e8d2fa3c5ad4acd619" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261027T120517Z", "generation_id": "gen-1793117084-CBkgW6t5voJpgTFZ5n11", "proof_sha256": "68ca813e0add532e5373a899f50bb905902467f12bb6a49b6b8ba614356cdb79" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "NextBit", "run_id": "20261027T120517Z", "generation_id": "gen-1793117114-X1gm1DWmqh6Gbbd3k9sZ", "proof_sha256": "98c471df620af63669d025d8e28978525f878d6926f7d5a9bdfb616ac47958eb" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793117175-AgnCqqYqx0HEOykcwCA8", "proof_sha256": "c25a9dd4470c3741974fd0ebccd8f97e8bed83b1570f34acbb7d9722a78392dd" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793117230-FsSYwOm31efpWRbk1LZE", "proof_sha256": "b3e17d6b2685135f477b481813ebc83804f35d3871b9a8e41a7f14d6f5fe4323" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DekaLLM", "run_id": "20261027T120517Z", "generation_id": "gen-1793117275-juUEu5Ak1Csrx82lVBm5", "proof_sha256": "ed7bc530fd536c71a75cb50f0c68183bcef10e2b43c1dce506899973c73b77fe" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Cloudflare", "run_id": "20261027T120517Z", "generation_id": "gen-1793117312-8CRKcdkVUtbhrvTnPATQ", "proof_sha256": "a105a85f1d401c3b0d04b0829568a4d33793c832c26830da1322ca4e7f1213b0" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DekaLLM", "run_id": "20261027T120517Z", "generation_id": "gen-1793117426-bngFcusdGXNNB2Y5nSVx", "proof_sha256": "2dc7bb6eb71bab469fd75b30cc87055960be8cd784c84caa97de7bf02f948f6c" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793117501-8eToOo1bfixCkp6dNzQv", "proof_sha256": "c6a325cc8e06d096eec2ed9b944568660cdcb52f7c5f855ca84de269fd591f24" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Io Net", "run_id": "20261027T120517Z", "generation_id": "gen-1793117535-3e2ThJk05RpkSYs5fbzC", "proof_sha256": "6878dcd2e11fc9273cfbade8442eb0db28f09d770d9b374c9f15351cad20ced4" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261027T120517Z", "generation_id": "gen-1793117594-91ybNcqQK17Z66l0W7Xb", "proof_sha256": "b52fbdfa4b002fef10b03292f6866eafbb60aa6981fbbf014c23195a85ebee18" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DekaLLM", "run_id": "20261027T120517Z", "generation_id": "gen-1793117620-hXYusHRID3QjayobDZ2r", "proof_sha256": "2e671fb8c53a1289cd4d2513dafc99b5ec7be73dad62851d92720cc51bc326eb" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Makora", "run_id": "20261027T120517Z", "generation_id": "gen-1793117675-VAScSGT3qqYv7DgUdqsW", "proof_sha256": "863a092fa3116b20e7a573b26789a5cb787b5755068fff7c2171a652ce678681" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Makora", "run_id": "20261027T120517Z", "generation_id": "gen-1793117699-w4iSlRExfQmCCrbbbp3e", "proof_sha256": "e6dce732bd3fdab75b2d37027a30cac17c4b96ecaf1785b7a98860bc253b564a" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Cloudflare", "run_id": "20261027T120517Z", "generation_id": "gen-1793117722-TAWmk8eSep1Zi88OCcV2", "proof_sha256": "efdd4dd23b055c50941c482ec4b0026aefbb0236a345846cf567279a962c628f" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Makora", "run_id": "20261027T120517Z", "generation_id": "gen-1793117858-hr5aWN4wwYqRd2o7M99P", "proof_sha256": "d3d95019f53e29466889764202560bb267c3c9916aecd09c68e161804bef94b0" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "SiliconFlow", "run_id": "20261027T120517Z", "generation_id": "gen-1793117888-qCEjuEVryOVPCmh4fZeH", "proof_sha256": "f596a7f69d60034dc082e55a20aade29aae805f542dfa350aea1028ad6c15206" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DekaLLM", "run_id": "20261027T120517Z", "generation_id": "gen-1793117930-ITyZUtCiL9LHDxlQjpHq", "proof_sha256": "34c019996370a13014aab90be3a5ef20f17362dcdd02de981394066e51059274" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793117972-SaE4rZjGh4yerKy8syhR", "proof_sha256": "2f41e359e75af0c030cbbe29c97d22090bf42b5a03e0a09b5c452e446aebf791" }, { "model": "google/gemma-4-26b-a4b-it", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Darkbloom", "run_id": "20261027T120517Z", "generation_id": "gen-1793117998-DhM87gonFpVJIPxTQoMP", "proof_sha256": "7626e421732a665cc8f0400b3514c6ba9da2b1508980a28789ae8977bcac4155" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793102717-ka5PYDHhk4R2symRpQfb", "proof_sha256": "45857e94fe09ed77c68f5b40fc6ad6ff897d1126a55d4d766efa4e359724b6ae" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793102782-Pyxbo6GodwapVtw2tUOE", "proof_sha256": "9abb55f23277e90cf0b5e883cc9c22d42a0646d427fd902bac1a8054db83c129" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793102869-lAnifgaZjLcPjwj0yxAc", "proof_sha256": "46605d860158d31cb43dbc8906f35a42fd99934009be6bb3cb06bfd092d3201d" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793102949-FJWzR0eXhiHu9TMs68Rf", "proof_sha256": "935759af5e635cd318c6c2043489ac3db711cadeb9c92ab6888fcaef6284960f" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793102999-UYrIsUvT9FB8CqNQRNDm", "proof_sha256": "d716262d4933a0f1c640cfbf475f866dbc7f1f68be5f57369ab12ea360d436fe" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793103151-XYgf1OeEk01MXtoaS2LD", "proof_sha256": "b1ed70fd2c4451de4c94460d937590407fb6f02c2692ad0ec1e45b4400ba5ead" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793103233-lrxdpuT1AMDm73IesN9b", "proof_sha256": "86704b1052ccf3220b3ce5a9eb9b909e92296de7ae89771977540bda7208cb5f" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793103299-zENYilUqSfSKhG4f8euy", "proof_sha256": "379723cf77d3bdcad269d50549388dbabb46d7287aa9341789baf69c16ed7664" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793103392-e8qDnRBlfQbytHZ5VSho", "proof_sha256": "304540d550405a230ac93c92b5caa61118c41da24fdc21645d92375fea554387" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793103490-DTKmtLYFFNjg5ecABUDa", "proof_sha256": "94307fcbda8d08e307b6446795b721e9b39620c45ab61db605d8ddf263008635" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793103608-9z0nw3f8QhAUnfTITXEl", "proof_sha256": "0149ee23dc2890d4210e45fee5beb567d24020eab2a3062b5d56aa610ce5f010" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793103658-uDlWHAZB2bYLh8wJdtht", "proof_sha256": "57c9da9f70593c66a8b8b30fff8a4bae562654b6c810359de004212a5c58977e" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793103747-ZBIeRqDWjEUBMswj9VLm", "proof_sha256": "16ec17e6fe6d06e1ccb620102796f9b44ae7d1ebb246fce837bb27e11cbda12c" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793103809-EPRwJkwdDRc4M9hCwbDc", "proof_sha256": "0ba26168cbdd14f53ef36f6d4f77324c2dc158d4be3935ccd2d12f514e1edaab" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": true, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793122291-wHSqOfyhL0kyPlujez2w", "proof_sha256": "ea45c613f132ea6fbed64e82647edd838929ba2661fadaf43d0b24bd5e1040b9" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793113972-x2rdMk3b20zXqtnK67Ze", "proof_sha256": "30502565605148a80ec51b0ef7fff935d23d711ea643773ce86aa70197907398" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "ModelRun", "run_id": "20261027T120517Z", "generation_id": "gen-1793114160-DP6FrYvEZ6fw153oOPDW", "proof_sha256": "ac3697de381b13d9bdc18847de26ca91677ae52d652f01e6e6e80c5fc08c3ae8" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793114197-tEQ1yrUrxhHuGwXP9u0I", "proof_sha256": "99a6b6c34aabc602b54952e1e5facaa278a03715e22eb7273c1e10d08d13f0d4" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "ModelRun", "run_id": "20261027T120517Z", "generation_id": "gen-1793114517-YM1j7TomaN4OPVnkkySX", "proof_sha256": "a2580d981601c2f6c9739d4f85a5ad26499a0c618f09681ef32e09bd79eea358" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793114548-t6Vo3rTdIGUv3Q293ppM", "proof_sha256": "a74f729f92eb7d2c614e439b1b0a23afd843596e870a0bff73887e5decd7f25c" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793114650-ebTnIO3RyuqZP4nkkZLr", "proof_sha256": "5fb5cb639b2ffba2705180fa18e4e5229711abeea2fb651a58076a0f52f43bf8" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793114811-iTbH6ezlCkrx7sMw2uY8", "proof_sha256": "bd4eb596279160d1af8093e11c596c582136293d8223fd5e06e1e1a03e25c4cc" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793114970-WFeIYW0kf8IGmbtvgcuy", "proof_sha256": "088a887ea58bf34c8cc010ac652f028ce807ac456a929fd19343a4ddd028f26d" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793115159-IV1fJoJyJ6u5YSSG5vDh", "proof_sha256": "26d52fc960e04ed3a22aa5e53b6c95565601956ad031bf7c02c395098d6f212f" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793115215-T1CbXYRVLSIIm2QwCvdq", "proof_sha256": "d1ed314bb709d52fd37dae5eba0c52d2e73caaa8f53a6bbe52b6062ef47d9e5a" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793115355-fsl5Ofvm34mLEVoLkwQ7", "proof_sha256": "a01295b96d608353b0f654b01c970ed9410848602ad57ad9e1ab7dc424db3637" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793115430-lirEcUTNoyFLfthSn2HK", "proof_sha256": "b93aea2df29653e3443152064022ff1987d2f11863408d461a5923a282755d8d" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793115554-HBOdu6Bk2FU8oKRT1mxW", "proof_sha256": "87165826d574c6d8b38bb144ddbc370eddf63d432e8f54d7ea5d91fdf42d6883" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793115663-bJFPZ2pEi0N6HQx8iF7a", "proof_sha256": "b6ba508dd20f1c490bb706dd13cab7bbb5834351b65dabf3691e9ee22bb1a8ed" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793115743-JPdyTvTDTqQ5TcxoNWOR", "proof_sha256": "e40905c5fe6db60d8069f971a4f04bfe3ec710105c5e968f10e1c3fed3790a23" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793115824-GNviBtukvYhAFXUI8XNO", "proof_sha256": "c2326f3a6fda4986bcef10cca8fe4a52828aaf0ccc16224fe9a7782f4b43a514" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793115897-ziDOYBsCpLcnIyXt6wOC", "proof_sha256": "d5888d96b04e0543c5942e86f46b6616640c90abf4ae5455125823bab1971ad6" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793115949-HUU2Xl3MI02Kc1yCpkt1", "proof_sha256": "79fd8c459595254c144e057af1a8c1fc47e8d25665226b322daae0d0fb6428cc" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793115993-pwUo7WrPHdg8DEPGWhHO", "proof_sha256": "74e07fb33ed282fefe985c97fea1b821870200d60a2c4a301df64a0f0a4eeb90" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793116062-FrpFAkx3TFCUkXXUzEFz", "proof_sha256": "fa3520855bf64f9286e02897e09efa4fd28a297c4eb1d02ce821d2bae6846970" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793116133-sS3gkIkYBNFqgTbpcScL", "proof_sha256": "89cf25d79810cf3969de41bfbeb2a597ce102078765a1c4df21a52324285723d" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793116210-EQGXywP5AcVgzCGM1hPL", "proof_sha256": "d57df0e9db2e79f523c0812ab4e46a7096b4690fa220d3c040d2e0864cf2771a" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793116262-y8FFPkeq3npBkpp3fTZl", "proof_sha256": "95e9be5e6393007095d56e54bfddf5be5fb7d208f93357381c35defb3e09a879" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793116358-QMGjeKikfZ7NZhrqapQL", "proof_sha256": "14a58fff30b731cb2c5ba5a87dd52858c2d9c5ff65272aedbcf88bc960c69253" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793116449-Nhu2YgtqBXwYxxvS9aUm", "proof_sha256": "6ef38c27902df32623b950389c733bb0b6fe9f4f3667981ebd7fccb4e98be2a9" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793116497-w7tbiBgJdPoR14qrPucj", "proof_sha256": "18d2aceabe912ab86c01031b46b21b6978328477d78343e94bd33fc6e573296a" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793116554-agyYA9aTOlFWxQtgeS7Y", "proof_sha256": "6ba237450d70a032e2c51eb3753eada06334166b8b2493bd7df7cf84f6d770c4" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793116616-lUfNF6iuVOkltJWJS9zb", "proof_sha256": "6bf29ae951c0a7fd9a8e8455826edaa14694ae0e5ecb4ab109f9338500726ebc" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793116689-vx8OLqAWHyK6UjS9dunb", "proof_sha256": "565ed8646677deb302944b2415fea2330b382f72aaba08df8a161efa36667aa5" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793116728-YhyntXNL01P8wtMz6czd", "proof_sha256": "bca72dd95b67a82996361ee1b8a9ff39c2431f4527be616687969e1b12812886" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793116768-QGcKSNHSKKN6hD6WfLYr", "proof_sha256": "da575574a999304030b528a3c2ca5750c38a07cab8982382fd219e677172c446" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793116836-iSo9iiJjH6xMtUeN8KMD", "proof_sha256": "66248d30135671ac9d23b99dbd3d3790c28548270898ad9fa3ce89633d235738" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793116884-t4kDL9IqD6KMjjQrCdLg", "proof_sha256": "b90b3f75ca76d2e72276658cf7b66ed3751d0f1adb7a7281c9621b3937e54282" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793116967-bv3t09OCredo5W7YGor2", "proof_sha256": "85ba70b19a2e5a19346708e43bf5b8938993c95ec8b004179fe50eed7faab151" }, { "model": "google/gemma-4-31b-it", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793117052-4GshicwvRx59QpJfcCc1", "proof_sha256": "198f74ae597ceea7f5639e56a296dafeaee252d350c17a01d7bf6dafa504186f" }, { "model": "google/gemma-4-31b-it", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793117107-0BXXuyRDHB6cEWdRyiEY", "proof_sha256": "4d880803edd8d4c45364216006557fdca9c58607b886e7f5104aef42829a9789" }, { "model": "google/gemma-4-31b-it", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793117167-QG8z7ndRkGsmLMRwlfQ2", "proof_sha256": "79bce8b4cbc0361e99353af6053c9d2b81625d7cd5f89373a0a949c3838db6ab" }, { "model": "google/gemma-4-31b-it", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793117261-qpArjWehBOSFKvb2UhXp", "proof_sha256": "6bc1761d62b3eacb90838aa115d0fc55b2f955d1b3f1c56d38d559ca79c10459" }, { "model": "google/gemma-4-31b-it", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793117366-2qQIgh1Lms7qQH1uZBcM", "proof_sha256": "7a177c76168c7ad39aa33b58642bfc00f7a0e6a534a8ffaa4b67634d568cd0b2" }, { "model": "google/gemma-4-31b-it", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793117439-ouS6jhu9xqA6QrR9x1yU", "proof_sha256": "91c29c09c3567910db9e1fcd3d81cd50e627f86d017f890f36906716ef694942" }, { "model": "google/gemma-4-31b-it", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793117527-vCva9biT3NIev4fUTEfz", "proof_sha256": "1c8fa42c39409c1666a70969e4cadce042f856102caa88f281d3f46bcf622507" }, { "model": "google/gemma-4-31b-it", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793117563-XsM7Oitf8i8i3dcm0eod", "proof_sha256": "29d41c48cf8988c028638ce9d285780e62e2160445ae09a72c28a7cafc7d43d3" }, { "model": "google/gemma-4-31b-it", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "the correct explanation for why lizette cannot see color in dim light involves the different", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793117619-rssAbIUbx40y1udMmw47", "proof_sha256": "b6c1931f5948ebd18e0437a8b76836568b0d30a2ce28f776c696cf103b6464b4" }, { "model": "google/gemma-4-31b-it", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793117654-Af3cLntDkSa3WgKC4Gx6", "proof_sha256": "c3def87976c7c359950380e33b507236fd94e3f8c399055cf316038b3565e76e" }, { "model": "google/gemma-4-31b-it", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793117712-yvNA6qQT0vhJCb37dyWO", "proof_sha256": "2d97236749de4d4173478768f710c5a1514254474bd0bbbdf57fc5cb0fcb69ed" }, { "model": "google/gemma-4-31b-it", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793117758-Zmwz7O1535UzJrx0RoJN", "proof_sha256": "db35b17931f2ceae993e037f49d765f844a07aef7838a1eb331e861a94bb03c0" }, { "model": "google/gemma-4-31b-it", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793117801-x0FUWxgiSiroFWbo469S", "proof_sha256": "b4c7530c4ff95e9bb0b52fe99998e6dc99e14e6db2a96cce43f1d0213b523360" }, { "model": "google/gemma-4-31b-it", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793117827-DfCQCD1E8YBpHANHI4t8", "proof_sha256": "70b264cd7a0d170558df60f4c022fde8bba8490c1f3bc6f9a69ce7d33b7e0da4" }, { "model": "google/gemma-4-31b-it", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793117894-ZhDcsz5OwbIuJeBDBekz", "proof_sha256": "9f88b0c1c2745f0eef149098661ffb2ee14421c40883797a5580c0bd869ad470" }, { "model": "google/gemma-4-31b-it", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793117933-7V29DPuQNunIBkvfXjLU", "proof_sha256": "eb7bb603583f52c641c10bc581fa38894f7d6bbf4063e1f905c00e18d5b98152" }, { "model": "google/gemma-4-31b-it", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793117967-DSZyRAJTjHVd4pfKl7tr", "proof_sha256": "867cb189c19c1ea44e8b6a5f37bbe6d7c4dc39c9aa9236b8b003080a3c92369d" }, { "model": "google/gemma-4-31b-it", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793118064-L26Kg1XrQLrV5ATnIYRf", "proof_sha256": "ae6a61d0c99d21515862dcd19265c90b6a94e00765bca420854a37a3ba391e52" }, { "model": "google/gemma-4-31b-it", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793118166-Fpyb9uRraiglcU51EVgW", "proof_sha256": "585a8f950f89cf1a5f6d75e37d8b2f96fd2d7015ed488d707b9de648b1923c0e" }, { "model": "google/gemma-4-31b-it", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793118286-V2IrSlyPEnusggbwT5Xo", "proof_sha256": "8cf66936a62678136bab42ad68f4d9842bd2f7067f89bd6ccd2bced8483b04b5" }, { "model": "google/gemma-4-31b-it", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793118323-kQQUxzauLbbDuvtruI7N", "proof_sha256": "5f9a4ecbc1b93ec738641c175fee55487d1874f20af87a8610bc1ef7c5e8f014" }, { "model": "google/gemma-4-31b-it", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793118374-BULpRJeUnY6AD12j9daD", "proof_sha256": "fd52dce18c3629bc717f7458a4a537aa13083de5cb101853c6c7ede60e3047db" }, { "model": "google/gemma-4-31b-it", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793118398-021e9X9cD1fs6xNCBtPj", "proof_sha256": "1f356ac7d68bf876427b29497e87cac56202411841d58c5caaba3b45820112a3" }, { "model": "google/gemma-4-31b-it", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793118521-L17xFuIWH557d95MWoyt", "proof_sha256": "3f4a4f0999dd8affe25cf98748f9380a5df5adc240c5f3e839ce11db197e2e58" }, { "model": "google/gemma-4-31b-it", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793118586-pTsGA6vLZBwpXjt3oPOR", "proof_sha256": "0d61ad7c9f57b609074d9c79d244688438ca9ed1bf1f8aeab08db80e193f12a8" }, { "model": "google/gemma-4-31b-it", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793118875-lBrTf4wczfXTFhpDcsKH", "proof_sha256": "6c374b913a8155db61a000fa0dd454f38228bf276363e97a2d34e385dea5fe7c" }, { "model": "google/gemma-4-31b-it", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793118934-pTNWgr26yIT9u2phkQRM", "proof_sha256": "3fea3da9a06d0f461e8bd7fc97cc0e3e822b01f44504a7e439703c4807a8ca34" }, { "model": "google/gemma-4-31b-it", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793119172-exqVShkUidb4U59383fG", "proof_sha256": "8876c11a807a5c22f2fe30ca8c8fb06157d7178fd3b89bab7b107dd7de956090" }, { "model": "google/gemma-4-31b-it", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793120845-NOIF1ya3qxxiMCuldTTB", "proof_sha256": "0d0ea65fb5b980be498c600f58565499eaa070f7e505ac4b9f3ead2f47c30875" }, { "model": "google/gemma-4-31b-it", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793120927-ahvH1F55MBbuXEi7WCHi", "proof_sha256": "a0d7b125d783e1e1792b476ccf539aa43f754e5a8ce00252b312fb2e4598ae9f" }, { "model": "google/gemma-4-31b-it", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261027T120517Z", "generation_id": "gen-1793120965-8Mw9SNGN3rC1p3P1NDdm", "proof_sha256": "f3baed2924639b9e0cd60b8a68c248d27d1d576a0759df39156a7fa2a7132f20" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793102717-22G94IQErqSlrGOCh887", "proof_sha256": "b4130af1fa8be02e888cf2d222e13f5512a2a004fa0b60085cdfd9d19c00cbdb" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793102835-iGw7RHVO6QLIICaPCWKH", "proof_sha256": "eb25982e7a0711fe34d876a0e1eaf05066bd77e6d0b2c955dfc72ad452c1223b" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793102908-F6CoCtQu9zg7q6h5Oi38", "proof_sha256": "97f5e98a15ca1923999f1c42351af9cdef557ad491128ea145ca3f0ef4bb3272" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793102966-4ex35cGqUzNk8zFeoFON", "proof_sha256": "1f3ab267a36f7771aca899eaae54c305ad1898a35256b04a31a54eb09421b7e0" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "taking $a=-2b,\\ c=b$ (e.g. $p(x)=1$, giving $a=-1,\\ b=c=\\tfrac12$) makes the top two coefficients vanish and leaves $x^{3984", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793103038-Np8GDHV0tCcRVdBsQqqp", "proof_sha256": "0db426d66baf57e7d1162ce96578674007f6713cb36d1638d21b6cc88ce82c9f" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793103230-TtcLrCCtZPQdwKSUz4jz", "proof_sha256": "672806b53d345675726bc668dc8ba0d6bdf5ee324492215e603328abaa9bc274" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793103268-lopcZTtX8hO2CFrHkNKi", "proof_sha256": "cfd89d8db2bd44fbd051b86593d508d3023202aa607513ad1d18ebb83b612d87" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793103315-E84PZhFbT5EPmRIRJ4nC", "proof_sha256": "42b3865242ace39b299793a13d1fb51a14114d94e4a70d096f6fc9667e00b8b8" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793103479-P9rackmhhPZamg1eiX6X", "proof_sha256": "2b22169ca63e096935acc40f8da1d98b0d4a7da3fcfb2c87c1de883cab1f0a57" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793103496-Hlb22dh3u0dEHQ4aVyix", "proof_sha256": "c999352ea4d0bff3bf0b3dffc2212531cdf87ad6e57f424728c74133a3daa4ce" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793103686-J2IECNiwJnMJHg3jH6d2", "proof_sha256": "0a06fb297abb21f598f48dd02f4e19042ff20a0b1b14067d08f603bff86fea70" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793103716-C0yvHagUhFgkSC01F1WP", "proof_sha256": "f5ac76238179ff4ff989e9a2f2b8199a6e49e07b487c7a5209e6919ed5fd3203" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793103757-8g8KBRbQDheKPayib8Yw", "proof_sha256": "19e323a059d76462433aa2a6cc8d1f3c509d9faebfc734a7c181834552d33896" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793103788-bmCRhgdTAogTyorFLyQy", "proof_sha256": "11442e0b4453308639d481cd0fb22fa89bf0257ff1163f12392faa947be7ff23" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": true, "finish_reason": "length", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793122291-NlT0q5d0MbXwKyhV2BOl", "proof_sha256": "bf1529dabd27523b7abe88efc70843dc0290c8267a3f86efddcae8189b9d712d" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "$$e = \\gamma m0 c^2 = (2.023 \\times 10^8)(94", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793113930-WIcD7mJI1tlh29DTl1Cq", "proof_sha256": "fa86653a4dec8a07ab1bdebc39c58bcb7cda261ad22961d9c363c6bf1cfed877" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793113969-hz21MVgvQ7O6ZEuyhTYJ", "proof_sha256": "46aa260d31133cf48c1ef9b878581a30f999db02901cd9e57bab22fd64bb90be" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793114207-9XoZobirjtvZnqr075TU", "proof_sha256": "b60c6dd671b447f8da5aea0d289a435300e042787a3656de8fc8b2eab614eb70" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793114251-PbmRoCb9YqBVwP3M3YP1", "proof_sha256": "88eb259227702211bbb2013d4c17475a3dfab9b761906c0ba2cc4033b2452d09" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793114276-1IvwZx1Ij1tUCgnSj4RH", "proof_sha256": "173f6e51a6dfd446aab3e3d905db967f5946ee159c419aaff9d9fd015165c8e5" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793114341-VHM5wqf9WXkhOSah4iVn", "proof_sha256": "9e6b6349da83fb85f2657937b37233802a785660f60ed3d20178c1a1d41839a8" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793114393-wLupEPUvn6D5hxByBO5E", "proof_sha256": "dad2baf112653610eadd46aa1d1eb8864789698d9572aa44b1f0e109647d4e8b" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793114411-NPsEM93vYhVzGDWkTAjO", "proof_sha256": "74782fa58b3e98bd8ffe9e15c1b28488555bc53f32b9a39e76196c0722aa37dc" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793114459-Ys0KPSfF6xVm9itkIrN9", "proof_sha256": "93484b3a2a722f972a7c12836eb7af4d2a7a26de9d9905cd2b039c1533ab41f7" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793114766-BDTL520RsHeADKZWH980", "proof_sha256": "0ef4a4e63b7ae9409524e7fbe8b37666858965047ecbe19984c0e9c8f10756f8" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793114827-PaibU1K54KkLWmInUBNH", "proof_sha256": "852911aaf2d5d7ef4f053f74086c53baafa874bb712814f1cf3b5ac8d9a28eb0" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793115143-hJZJ3moOkUwyN2Rj3Waz", "proof_sha256": "ff6feea7026d78e324dc1ce961dd2afbb933bf01c2e0ca844143489a4d0b1d2c" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793115437-ZMF78qLwHFRGSNhrGXut", "proof_sha256": "380142cf0f92862663f251f94efb268e2926a88cf1458a688897c985c0f95bea" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793115477-GqcMnm17BjfMSyF9XbI7", "proof_sha256": "d2be0e14c9b51f1769947cc8ccc33bbf71d1198462e6b69840c51a3a638f8611" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793115529-VCWuc7bXCjvuUB32aZ9n", "proof_sha256": "c6818820629ac6bfc9043c0f9920dc6e3cde4ac8196fd3fefde1d7fb02655d26" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793115583-FAhjkZvgBGJXEtXeDVZv", "proof_sha256": "8afb61cabce13282e8fd7bab0a675cc430e3be2f2f64b6f500068e321e6a0e28" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793115633-8cHE9dZzXPqgeFxUtTne", "proof_sha256": "3ed511c7be2e887b0a4509948435a068f2e50aa8f9d2740b519db98393ed7635" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793115713-I1Z3disIFIOU5l6wq30t", "proof_sha256": "dc2da775b0116f35233d2bc05881a9a8874a5442252e596e5e88e0054f014165" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793115777-eOuqvfbgaUfAtifuAZA6", "proof_sha256": "a847655693e93b0d542218922b7b43370b5f429467417098fc056a8d1a39d64e" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793115959-xc8Xk77FOAUge4xuyHdg", "proof_sha256": "68e991bd9710f8414364f79df6d8db57a256d9f4d408ac87c330b79f9a95d8e3" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793116037-5HYkRE7VJO2y2hoC3KUt", "proof_sha256": "8972858e0d312343441cc82fe3120c3d2ad2bb201e81e4585c8ea8a0f0252a13" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793116075-ALLryY23FCVC11KJWVUM", "proof_sha256": "818da9ee06abffe4e0369966c05aa75359414069394977501474a21bcee84732" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793116117-eENL3e83wMMBoz18wNrA", "proof_sha256": "136dfda5c447ed6d375b39b67bc3cb1e897be09c914c6e5f54e85dc4fc1f8926" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793116158-gdwyrHIb7fPEgdrNV7wm", "proof_sha256": "5b8d59a6a870d4ecd6e94272ce4558be9e490fa5c7997770fe86c1d1cd4d63db" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793116210-yIhUi35BOBXJ3jIO8OYE", "proof_sha256": "d2985d93e7dea72caa2b1d83d3e84d1923bd1f213008c5aa38f58cbcff3a6e8f" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793116370-YDPWYpfi6jyY1TTWWjko", "proof_sha256": "4a836990a4c37dbcbf057ef0b5c20032667b583266fe2286937b89d0e744beb0" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793116413-8dccbT0LJ0CTP81XqGeL", "proof_sha256": "17ccfd2fd53d6798ff10a4121ff6625e88f0eacff8597d0424b527ed76652235" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793116466-lpYAS6WT9wxVnqqKjMdS", "proof_sha256": "756020bb963320bcdfd7cb55f88b02f53f892aec6369435fbe19e3eeb3353e7f" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793116685-ECCmA2H7qmadR6eVr98u", "proof_sha256": "9a22b8f25ddcf96d3091b837e899b2666ab588d212d1344eab139f75b26e6a2a" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793116921-vrcg7GpriuEESNkJgkA8", "proof_sha256": "dc63d4dd313df8f8d606d45ebf3785a3b6750d0765e42ddd1e0c9a519e961130" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793117005-iZMz1mzFzJTlP0FQxuKJ", "proof_sha256": "0d73b2d21a101ce41fb4af2e567540462aaccfaa2fc998dcd09590f28a75fc49" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793117226-BY0zGVYndjinXYL4PDZT", "proof_sha256": "7866e7049073ff3ad1d77427622f02a3d26a592cb9ce51caa01a833990a8ce75" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793117276-ZxVJ2vmDBIcL20MIHgDE", "proof_sha256": "756731750d91a7337f6b2a9e59cbff262d4043fe997625db3f8f10830e688899" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793117332-b9XQ0qCuHF5AdpQsfeKU", "proof_sha256": "68ae9a0a241b9268ef93027b946d522bd0801e25752fba72b3d3755b2004fbf3" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793117524-aDW0RkeYipKRxPdCAYKT", "proof_sha256": "c538e4a4dd32a2a459e6d8d2376c2607ecf0e6fd03e6eef700222df5847dd6dc" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793117576-ZliJDQio3HVn6TKf4Ks8", "proof_sha256": "0d9fc44bbe974e25537e53aa631631c1fc70bb196fade534c3bfbe5168cc69e9" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793117785-QltAoKTcphfSQXJj2isw", "proof_sha256": "e0ee9ce3317cc81c4729dba1d1b40ae39395a29659d3a28bc533a0abd843ddea" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793117853-fUMEhkfT7Ap9wZIDsJfV", "proof_sha256": "a936e40a42e860244ed3f482922806a99542abafd2d437bb3b4164ba63a31197" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793118048-RMFn9fzEyRDXG6TM6hpo", "proof_sha256": "0e463d76e10a7fbdc3a5e26fd859850a2df6e7ada2aed534b53977a2a0d773f4" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793118115-hxYkGBz72uJVgktpyAIb", "proof_sha256": "4e35e0f6e68f16ab07db005ae59647975fdf05bb37627c892773de441ad3b511" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793118136-XWT4kn7bfNXO8bMKYlik", "proof_sha256": "b4d0194731f2ced80a7cfbf32ed678c80b1467fdbfbcc0b0b3a6b51d1f6489b2" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793118186-1jfdQijX7vibs07VAA6t", "proof_sha256": "db37ebe5716561783d2769489ad27294bdb307b97dd6df3fce5671f7c29c3984" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793118199-cBQuczdmde3XZou2fWkv", "proof_sha256": "b5e7306c5327e0a7d1523a80f9c76b485d3d80c60befe142569fb7ba891f19fc" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793118258-2R9YcXWKFYWNAPR8WgHs", "proof_sha256": "adb89f1372d5c716bc606174c5f917ae79b9deda891edfa0d4b9ed95ae579a00" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793118286-EUWNV7y0oJXEbZbJ5jww", "proof_sha256": "e2495afb7b0e6f8f8687a714beef57f1278df34509c88b896a7205f68efae401" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793118310-dwjZ2ccIbpPrHiOx0zxB", "proof_sha256": "7430563dc4c06b779aa80adb8101dcc6523f64703f7abfd2545d3465a97866fe" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793118333-bbPLMPVGxwIqSakHnPbi", "proof_sha256": "796e9b8f667a6f14b12103b9fcbf978a7548b95d17aefe762e0778130159a5b1" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793118358-nKAhM8dUye8WMtq9ZiUM", "proof_sha256": "da10388b5236c7a61a00a36f97e46f7b9fe43d74010d7c5d126995250243e235" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793118404-B5PAMkBFQV6vmYkpcxMd", "proof_sha256": "70f37fe4b96c14583a649e5e73ba719cbc6f27de5e7cfcdbde3d472eb464685e" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793118424-5tFLmHJ8K9Mcva6KikxY", "proof_sha256": "9c1914cc1ca4d0c26b1789deb96c57fd4a5e7e376d0781721003cb3224c86673" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793118444-ezRm5PQH6hATvCwZoLTW", "proof_sha256": "52ce5018653b8248c1fcb0dc92f4625c281afd63aa9fb726ebfb0484048d2d8a" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793118463-z6pvjefwDwPj28uMT1we", "proof_sha256": "bb0bf066f7a35ac5f724a5487e558b050fb0b64f1d902c8f5e6909c1a3149472" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793118485-3LsnqTMVzeketmT8v7mV", "proof_sha256": "18f0b1c962e1ee849af645fdba185d1e84c32a982205a90698d52bd0e7ea6f40" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793119175-ZtxycvWmstNo44ShXxNn", "proof_sha256": "6602f069c4097c4be41b9e54e6b3147cb6907fd57dfdb4f982e6dcaf7e9c0065" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793119185-QW7X1k2ga9QYmT2TMTPM", "proof_sha256": "e24c6210431f31fa179a2195d9823618e8c3c69fdeb9cb66988a6bda850e8220" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793119212-3TKqdr6ugSDpotAi5Pi1", "proof_sha256": "5143f72c18f49f538375af476df5edb85d739247e7c77dc1b0262f5564b41123" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793120844-gsDiDTe56na0lgiNSlsj", "proof_sha256": "c6c2d88ea1aaa52c870b6d4bbc310503faafc903b1e3534150de46b08f90602c" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793120921-Muc6uOxQXyfUNdwmcNFy", "proof_sha256": "923826c5bd2da79a236310a31618c7458a80d27523187535378be3e8d0e4c948" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793120937-dn74pilshLZs1ig7Q6XJ", "proof_sha256": "9ce35c49a8251413783a4040c07be250d71684b250704d52636349c85880fff4" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793121097-m280f1ac10Bb5icGfVur", "proof_sha256": "d4ea46b172d3088108e041bd115b528f4bea2aafa6e430c02598aceee49325e9" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793121120-wej0g1lUSoHtVTSlVowp", "proof_sha256": "f0199ae6fe0a81829b7f92799179ff665d14a5594b189249f3256c798aabb7fb" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793121165-f6kXIFRzwJBdbeibHBhy", "proof_sha256": "1f754157552d43360b268812a00fbf9e59e9ffac6734dd83bd51099a91b57bfb" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793121182-7MnGkK7g5f0Ul3sah1D7", "proof_sha256": "8b84b6cb3ce74dee7076c8d61032a8e7554a921fb320f19a935b186f9b4b8d80" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793121194-ric7ribTUpylU1TrAVqV", "proof_sha256": "58ecc84ac45681653a3a9080c1e665488e0eb21f44beeb668f713135e355793c" }, { "model": "inclusionai/ling-3.1-flash", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793121228-yyeK8T8csO4fDgXfurXx", "proof_sha256": "45ddf880a84ab9d8c172b0e80ac5cd07c1f74aceb704f9f191cb3508b4f9b6cf" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793102717-AcI4yWGAeipkMUCTps2i", "proof_sha256": "04d3b7c3168099507ba7bd684cc93335ffe1ef51758a34d257100269984b52a0" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "f. diverges when $|x| >", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793102896-Rs7rxmsYGvFMb8aIvo08", "proof_sha256": "052decb295ed030bb66c6835e70aa764536c5cc0be93758d7af19191dc716c80" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261027T120517Z", "generation_id": "gen-1793103090-sgfsI7o1XgqMOzf7MHwB", "proof_sha256": "b0f16408dba4dc1e1e26cfb25354d8bc515ab6babb8d5e537cdf1eb8cb3a1c0a" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "StreamLake", "run_id": "20261027T120517Z", "generation_id": "gen-1793103163-3fWh3LqG0pRcIVzwPuOq", "proof_sha256": "7d8b9d10882eae1f5c537c81fa7e1489e5407c23667020afed0b04a36f502d27" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261027T120517Z", "generation_id": "gen-1793103233-VLf3Zx8sOAVngBzGHeGo", "proof_sha256": "c1bf3464c5e75ccb8fd68f6616a74ae53c292f93f1947196747e23ec2fc93a04" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261027T120517Z", "generation_id": "gen-1793103300-bXORugeh2fvyOMvnwXWs", "proof_sha256": "d10cc37aeef2fbdc1e4aff9c06d0ce5050105821ab63f51c71e310ef096ad47c" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261027T120517Z", "generation_id": "gen-1793103352-ePzm6ymPL4CdJwt3PMtq", "proof_sha256": "93ab42defec64e1c51e7b09dfd5516397e17f2be975414e372c93ce4863e122a" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261027T120517Z", "generation_id": "gen-1793103397-4Q1wBbjM0scWZfLTd29f", "proof_sha256": "8966cd91f9eb09ef1d62ec8c3f32db7d2e18ee9dccb4e1b690cf26eb54af24d5" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261027T120517Z", "generation_id": "gen-1793103445-lz28G2tZb1aksCjYaRZx", "proof_sha256": "fc57f83459555637a370e739c55dff3e0c8bd5fdaf1158fc5688de6de4956053" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261027T120517Z", "generation_id": "gen-1793103481-GiR6UFMJF4FE4HF0rEV4", "proof_sha256": "1f75c16314d6c6f232a12df99fc2d4f45b9219feea110919abf7b3ce187cefb3" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261027T120517Z", "generation_id": "gen-1793103520-kEfghzIvigWv6qvzoaTt", "proof_sha256": "caf599c7da05401d9198b295cca03bbccd2aa90ba0a0ee8d8569fd9343d57a6d" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261027T120517Z", "generation_id": "gen-1793108288-R2ZqBrcUsWeQZJG36Cng", "proof_sha256": "d4a04943fc5be432a5a663cccb9dfa9e76ec2673610bb84021ec78becf0b7dcb" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "$$u(x) = \\", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261027T120517Z", "generation_id": "gen-1793113884-d8Evg1PP9W3ePU3k4y1S", "proof_sha256": "3aa2e3f17f0d76aa957b9054decc01099d81ba5633e4a493e83cfc6a73212910" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793113963-5WhF0wNIITe7Ap0D5vSL", "proof_sha256": "18cb53ea3b4d3f9a4c74bd0d5ed6be1cac105b5a727395347f20f83142cd5b46" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261027T120517Z", "generation_id": "gen-1793114109-vDy5Jact7HAGRKzmp6C7", "proof_sha256": "d5ce57a2ae6269e162ea03163d992a9a2530f30557c5faaf349d55b825bf111c" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261027T120517Z", "generation_id": "gen-1793114190-d1GgScalobBfaqQKQDjl", "proof_sha256": "cb350532dfeea29889eb3c9b7f0e2f2b9a419da39f02e1c437a078d84b8bc83a" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "GMICloud", "run_id": "20261027T120517Z", "generation_id": "gen-1793114270-e6mtrLbC0F544MZjztk0", "proof_sha256": "4210497f7239e58dfb7df2518e853a19e4ab1045b3f60d5cd43571e5c3d7cdc9" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261027T120517Z", "generation_id": "gen-1793114416-Dp0r8AnOJPfcuK3Q4VbQ", "proof_sha256": "c4674b6703f82af6a5d23e217c576e237241974cd0f550c83567825739a13021" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793114539-yn5bVHLCRFTdMiiIZstf", "proof_sha256": "240ffe17a46af6e88c4d67f64b4e68eb74c3a37e52ed2cc7692c8f694bbbcd0c" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261027T120517Z", "generation_id": "gen-1793114684-Z38Y0uUDEXd5FUSE2LqL", "proof_sha256": "37a27c116507c47b3e5fc06c47ef0cae65cc45b74dbde3ab886586e238de3fde" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793114765-y2iLPfxXvJLPBX5dyUiT", "proof_sha256": "f84bab92113c32dca02d39d6f6d89d6bc964bf8f7f3d8f158874509637560c5f" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261027T120517Z", "generation_id": "gen-1793114904-94XH7bZzJg8r3sVB11Qn", "proof_sha256": "48a0d104dbc9ac79f92ee02e50f9cf765cf09029819fe47e80508a81e23ff790" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261027T120517Z", "generation_id": "gen-1793114977-dqp7WwS31Dzgjrl4nzjU", "proof_sha256": "f38fd1932795716bc6b83da753604cef54c62e4e138fecebdbec0046e4a5bc0e" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261027T120517Z", "generation_id": "gen-1793115080-lujUamMaU5M7inzyVHkr", "proof_sha256": "07fbfaae28878278bbb1f25583aa02ba33561aa52fbbca94400edf6f925669fa" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261027T120517Z", "generation_id": "gen-1793115153-PBEacqfkp43vZyRlBXWo", "proof_sha256": "9d12f11a36042d84c53b4be9bcf7eebb079bc6fad35329eb46138c56323bd827" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261027T120517Z", "generation_id": "gen-1793115219-Irt1Z2iux8eqQJ6NuviV", "proof_sha256": "2d34070e0a86eff0e053a5ae649839555f3b21ff1583149347cb7bbc38278218" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261027T120517Z", "generation_id": "gen-1793115318-p0MirgFLXSCuhSIveW4L", "proof_sha256": "201e4c5711b78290ed01f4b954e7ca6b21871b73a81f5214497b25d76e213d08" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261027T120517Z", "generation_id": "gen-1793115403-HrFXR3ykInu4TEc0YjUA", "proof_sha256": "b3bf76cf87e01d7661a469ed250fe970d266dbb538c596e9e07ce65fdeb89634" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261027T120517Z", "generation_id": "gen-1793115487-sBNl6G3jFyet6U0hcnTb", "proof_sha256": "9546a0bf53b12fc4a07ccc5e1ea47f72a5883ce057dc15e346e4d5a637744958" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261027T120517Z", "generation_id": "gen-1793115568-vOYKrugDaEnJu1WpaBfG", "proof_sha256": "b786f49913c02b22ea154c84c5e42390b706de7db1595e4f86129b8fd54e302f" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261027T120517Z", "generation_id": "gen-1793115638-JFGJNV5Z4FLQmPnVZ8jy", "proof_sha256": "ecd6d5ea4aa1936e4161014671bd4c51d45316596c647e4090da1f30f60e92f6" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261027T120517Z", "generation_id": "gen-1793115715-FEgjs4BrBCJUwPoRVNL8", "proof_sha256": "099496d3bf41057af6c268b704407daaeb2b9a2bba6fa8a25fd5b1f97cd59927" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261027T120517Z", "generation_id": "gen-1793115811-evJYSLPaQGCrysSN6JJ0", "proof_sha256": "54757991e1405afbd42eb1b41f3b5629254e2400233564612961cd20aeaddf20" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261027T120517Z", "generation_id": "gen-1793115911-zvmzggHVqDJVnMORr5PA", "proof_sha256": "a2ed2bf5aeaee5b1d1db02c2785d7f7983055c4b1c6b13bd3eafebf98f0869c8" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261027T120517Z", "generation_id": "gen-1793116017-hODzzb9OxcaORbX9pTur", "proof_sha256": "5e9077e1b90482eab15013bae8d57f242d720aedbe7451d5ecc8326bd013fd18" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261027T120517Z", "generation_id": "gen-1793116090-ypijQqcgVjo1YuIld7Kq", "proof_sha256": "a89f56d58d49a2d070428f8490f3b59ba37f492110e03ae792255e457cfd63ac" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261027T120517Z", "generation_id": "gen-1793116198-g2SSmIbtHfVw1DnOZGoo", "proof_sha256": "85245e4d2c852e94396975473b2a2758159e02b6686de5f73763b8e3f281df8a" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261027T120517Z", "generation_id": "gen-1793116264-0JUd4ifFdHJAuUPWchFn", "proof_sha256": "4c3a07c960ecf3cfce844c47f0ba5810ec050b9a6f0940f974dabe41c76a86ca" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261027T120517Z", "generation_id": "gen-1793116347-pGtWzpLjSr3EFnTLp1bO", "proof_sha256": "7f71f17fc9aa1f9daa8ef75bcfaedcef10269203ab956d6fd91fc8970cf79e5e" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261027T120517Z", "generation_id": "gen-1793116448-HGd0UiGKJk7jOjB7viuS", "proof_sha256": "6df2d734f2de26fdc16db4a286affba40b6b7123d9a594aad2a6eeedf1cb8c3f" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261027T120517Z", "generation_id": "gen-1793116531-YJDXUG2ZkB8NSFlbsCrv", "proof_sha256": "b9ac0a752cbf742f2d662a651df138aef746c428bfe6e8e87ff8f11778568df1" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261027T120517Z", "generation_id": "gen-1793116608-yr65bB8uavAPVsD7XxU2", "proof_sha256": "eda00ecbca6ad7367919f5a37060dfdfa193e8313f0dc419499353cb1fac4c1a" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261027T120517Z", "generation_id": "gen-1793116683-K8VUPD9WWAwIs33C2avi", "proof_sha256": "5d51c754a2504267d12e5ab685826df14a3c6e0041a2f8df53be75372027c1d1" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261027T120517Z", "generation_id": "gen-1793116786-noaC1Yhfiwr5OXU5c4J4", "proof_sha256": "b30fc8139d567e3b951bd871edb152a828b52754e9805d4f5e0b28cd41884632" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261027T120517Z", "generation_id": "gen-1793116865-BilexYC4yittbfHCIjK4", "proof_sha256": "9de59b1c6f251224a93a8d210046840c8c119427c0af2df249ed96b2e960f47e" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261027T120517Z", "generation_id": "gen-1793116951-u2IWYYLFJ4Tl7JXDisa8", "proof_sha256": "0907b97bf4a178cf242ede96e5cd1203d55929f5d15a459c48d8037cd6def719" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261027T120517Z", "generation_id": "gen-1793117042-uqwqiNaQBAjkwQjneFUV", "proof_sha256": "f7c6e6bd57fb23f5c1342efc92319ca14dc0404b569b86c0845ddf209f541fec" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261027T120517Z", "generation_id": "gen-1793117147-pZOFmggEiaCAAGCvFUl3", "proof_sha256": "fd5c564d88c258b57ddadadf8b9bc13e2c9ae88a8fd65ccef2d05096e6a692d5" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261027T120517Z", "generation_id": "gen-1793117241-ucyf17mwNWZFy3MkX7nk", "proof_sha256": "a02f867a6d9adab488e1241fa86e48cd8c444875b2fbec31f5922adeafbfb9e0" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261027T120517Z", "generation_id": "gen-1793117338-HMee9U1oImfhFH9se4vI", "proof_sha256": "7242257820a151348d2086414d13745f6b73030cd5bfaa7c0bc0cbcdea36cb14" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261027T120517Z", "generation_id": "gen-1793117381-ZLIA52ZNNCZUwxeiVVNq", "proof_sha256": "922c2bf80e87b879b23540659d1f7e7032f5cf9b7fa9ea49b17db81c1e62af89" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261027T120517Z", "generation_id": "gen-1793117470-BbpOgDo6QQq7DEQlXhVh", "proof_sha256": "0b8550f65e2f3915c2f3a4531481bcb59b274ab0f1260cce295baffc25f8261e" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261027T120517Z", "generation_id": "gen-1793117550-oiPjs0nZOiULsg2ZSTwb", "proof_sha256": "5d16784a864c4d16aeccb9515b6939704f2a68e5a19124eebb17279584030df2" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793117626-sEqrFpbKGe2QCh5SEed0", "proof_sha256": "59a80c5b91ff93e4a577b4a013d3c6b0c4dd70296f4087d5bbb1be7946a1023b" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261027T120517Z", "generation_id": "gen-1793117725-g1msYFuiNjGCkzH4nkvI", "proof_sha256": "315d61ce61d94dee0fc57d56197f10626ae005dbcf688c3654a3bd86cccebaa5" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793117809-sncMVGlhUrUL9dJhVKyq", "proof_sha256": "51afeb5a249c7587e628b63a88a8320df4dd801b56eabf7cd2b66dd5e5e9ee5d" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793117863-Th83yU64qVMnysb41ksV", "proof_sha256": "1c474a872a4112f4754f219463ed500a9bd2cdc606d16d016527788e6784d4c4" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793117929-LIVXsUxn2LGTatm2fgXL", "proof_sha256": "bbf8910e687771918900e321d69123cd75b0f1edd373b904df32a36eeede3416" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793118030-u9ZKmbhmimSQMtps9h71", "proof_sha256": "c14ded6c813bc2cee673c0677f20f52da0570eaab638a04372b24a5caab4beff" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793118135-eY5ScujcNRtIlxxuUm45", "proof_sha256": "d06c367f3ab86df63e23f47c1ebc4a98720b65b2e7a6ead04a419ff737ae5123" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793118199-9IkYifzJHoCjY9t7IO2B", "proof_sha256": "5128ee6ed4c0d723d93a4d72fea843e555c936554f516b9a0657b4d0a47ca59c" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793118253-3R21sFhTu1K9tJAqZQd8", "proof_sha256": "c3741c8bcbbc664081fa8aacc71eeafb8faac0389af1ae654f8ed6bf61cde05a" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261027T120517Z", "generation_id": "gen-1793118296-xO7zE0KkC2JRp7VWuGIj", "proof_sha256": "e3035cd3712b11302b417169842331672f54c4323df3e8e62145f16ba1563ea3" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261027T120517Z", "generation_id": "gen-1793118361-Tf5PfKRVnRpQJSreoK4O", "proof_sha256": "cfded9fe658e68486e3bf472202e9fe489c4c0e952e7104b3482a4b658f406c6" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793118416-jRFTWUH7u7qBzqu32XLW", "proof_sha256": "8193a9a1deb9b67f1e09dccb209f3bf9e219e0e0124ebf8f25dc89568f0bbb78" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793118517-hlYnfsg9rh4wFq2q0yk9", "proof_sha256": "e475715afaca31d72d1bb3c77fe255b84033d97aae7875985dc5006b8e62e86a" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261027T120517Z", "generation_id": "gen-1793118875-z2fnlmtYmQ2l9IT497Mp", "proof_sha256": "b5937fa1220c72e4943eacd24301564ee7bdb9985236eed9fd105d8a47670b40" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261027T120517Z", "generation_id": "gen-1793118943-wXFfzfZ2YgoyE6TvnfPg", "proof_sha256": "252dbe91c5fefd37d2a4fb545b6c162e28f5069f72da925b64b8628527478a0f" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793119171-2IBYfer9GD8Eho5El7eT", "proof_sha256": "6e8103aad9f4dec8921885b7362cee0d5421f899485c589832c9278d0683a46e" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Phala", "run_id": "20261027T120517Z", "generation_id": "gen-1793120874-1pLbkyyRhW6OrB842zUs", "proof_sha256": "44cbbda971f774ca35cc70b44330e3637dd72ee6b11e6c589472f6509042baac" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "GMICloud", "run_id": "20261027T120517Z", "generation_id": "gen-1793120928-MdzuCnTf9omygbIOJF5F", "proof_sha256": "7109968f6437a6b976a9db5e43b4782ecba7412c5fc2d2349d58e94607b4c865" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793120982-BB9X5DGMtkP9qQJt2To9", "proof_sha256": "98b590d405fdc32a582a01e96cefec6eb4800b1ec80b22967341b977e80ac5ef" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793121172-mg3cO2Gb70mUq5yO3j2X", "proof_sha256": "4fb63bfdeeabb8a77bbef6c503eee5ac74aed3221bbcd7b97f951c66b4f7d496" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261027T120517Z", "generation_id": "gen-1793121204-4YqqpQMPTQju9CftBL4B", "proof_sha256": "6543de1987a3284cbe8ddde428aea0eaad6644cd13e0e194cc34a9a547da706c" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261027T120517Z", "generation_id": "gen-1793121284-oB0USLZwLCw1N3IwSCNA", "proof_sha256": "92f9eaf4554e377b5a2e5dfc47f56d0abc74880ccb326b3bb68fcfd038fa890f" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261027T120517Z", "generation_id": "gen-1793121456-eZNkbBq9y5Yc58NY5d1J", "proof_sha256": "62ad5d12fafb68b23014af58ecc0675043190c738e155bb5afe845243d73f29a" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261027T120517Z", "generation_id": "gen-1793121570-jNOJqblfQnwvnGYSuDFP", "proof_sha256": "8f1b38df11f4bf72928db67690dbeb6108b011f6dd01b5f8a3b78aea6f8a7cf1" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "StreamLake", "run_id": "20261027T120517Z", "generation_id": "gen-1793121658-UzyAMVJtusaIQf5WzLXU", "proof_sha256": "0fd65af192bde2e6313e38dcc387bbce4eaf5c9bc4bc65d39cba1aa62106065f" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261027T120517Z", "generation_id": "gen-1793121685-v66GhOa3a8324RVLf3Jq", "proof_sha256": "5eaf60b203972e48d2481f3ad573f2a6ac0e98b5a09b9e6f20cb038e621d3f98" }, { "model": "qwen/qwen3.5-397b-a17b", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "StreamLake", "run_id": "20261027T120517Z", "generation_id": "gen-1793121723-o9qaIOEQBjpAe1EtWp1y", "proof_sha256": "66868227c6e3bab3ce8bf8a86d3d02facf1139d6f09602a6d18f9dea12a20872" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "for b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793102717-T3GVb60ipGDn5G3b6vcn", "proof_sha256": "aead5f700541eb2398163c589d18b1a29f34b308313bc6eb38b6b7f4f47cd908" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "but now, why are", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793102854-36weLlKxydDPFFsCK6Tv", "proof_sha256": "083205bc8544088506a58d5def386898ccf130cd7cf0f01188050c5b4788b539" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "h. \\frac{\\lg\\left( 1-\\frac{v} {p} \\right)} {\\lg\\frac{\\delta v}", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793102982-mNwxqLZEK4U1bFJOmA9X", "proof_sha256": "5e3f7b4b478975fe91c0dd09eab0442fc18ad99499e6f297061c325976d925fe" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "now \\(\\theta = \\arctan(t)\\), \\(\\sin \\theta = \\frac{t}{\\sqrt{1 + t^2}}\\), \\(\\cos \\theta = \\frac{1}{\\sqrt{1 + t^2}}\\), so \\(\\sin \\theta \\", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793103107-UOOnOoWxGxK0ac1ctjot", "proof_sha256": "d9bbf8685f1cdda68e92ac1f606b64e1aa47195bb144cc733e7e18b55e6067fb" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "to make", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793103231-TsqKP4K0m2nRaFJNwkEj", "proof_sha256": "419fe7c51ccc54571cb542aad28773a1d2b566a3e16ea8fea57c069ede7782d6" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i should find the", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793103344-xbbHTUn9FHpxz6sxg0s8", "proof_sha256": "87ded32de57dcc6e40c3c8f8043efa17eecba368071a4095c27088211ac52c50" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "perhaps they mean that nh₄oh dissociates as nh₄oh ⇌ nh₄⁺ + oh⁻, with kb=1.8e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793103448-rFqd5zhKBDxo2ArRMNFQ", "proof_sha256": "e15e945513ef28c6dd5ebb6f7843ce971d7a58114c225b30e4267eb32c0f2826" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "va = sqrt(1.191e9) = sqrt(1.191 10^9) = 3.452 10^4 cm/s? sqrt(10^9) = 10^{4.5} = 3162, but sqrt(1.191e9) = sqrt(1.191) sqrt(", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793103541-aeLSvwdqHDEkfXKtQvx6", "proof_sha256": "d6eb41dcaa86766beee6bb27727c95a72c50fbb7a9fc56ce5d40da2f24bb5d7d" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "right side: \\frac{d}{dx} [c e^{-x} + 2 \\sin x - 2 \\cos x] = c (-e^{-x}) +", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793103632-n9yQHg3ZUVz3i1Wpbkwz", "proof_sha256": "b9138224e23f883deabac6c98f11b7db97738c654fe7da60d522c7135b8ff261" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "³√4: 4 is 2², so ³√4 = 2^(2/3) ≈1.5874. correct", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793103760-TjZDmXrUwcnH1j1UL5E2", "proof_sha256": "2a2860420385fb2a3a4c813e5e0f52a383ac9cfc367a2a9728d97b95f8fe5c6d" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "now, f(n+1) + f(n-1) = [f(n) + f(n-1)] + f(n-1) =", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": true, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793122291-fVA4qHP2mF8gpHDmjcnG", "proof_sha256": "e8eac883746a532de777bf1493d30b83ec468560f71911f3c99d069c2704db3f" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "s", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793113930-I3avwYQnpy5rm2C7j1dH", "proof_sha256": "d4b0e30720d4d7b1d79dcc62f5ea1cce4f1f5c56e49faef7ea87c9969cd22b68" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "the potential energy function is given by \\( u(x) =", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793114046-L9f5kx1aFUP6u29qvvi3", "proof_sha256": "13b2fed528b19307b55c102c6877b647a17fbc556e3d47bad1a9a40febb8c48b" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "the last two terms: (a/4 - b) (", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793114159-zgmf6MvObRz6r8wt45xb", "proof_sha256": "f93df00d1e4f1e587c4906099fac697111dad3e57a96dc5feedcdd093684ca83" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "but wait, in the expression 160(product), 160 is 2^5 5. so even if the product has only one factor of 2, the total factors of 2 in the entire expression would be 5 + 1 = 6. wait, no. wait, 160 is 2^5 5, and the product contributes at least one more 2, so total 2^6 5 ... but no, actually, the products factors are multiplied by 160. so if the product has, say, 2^1, then total is 2^(5+1) = 2^6. but if the product has 2^2, then total is 2^(5+2) = 2^7, etc. however, the gcd is determined by the minimal exponent across all terms. so if for some m, the product has only one factor of 2, then the total expression would have 2^(5+1) = 2^6, but for other m, it might have higher. therefore, the gcd", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793114270-j1lh1YuXyET0ZVd4ynG6", "proof_sha256": "5d9920f4325360e7758a3e075c787fbbd13fe6289de44edf205fd1d110f13100" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "780 ÷ 1.5768", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793114373-ObhS2p9Mky5IGQCvauSG", "proof_sha256": "9cdeb058e194df21f90017e95a92c96bdbc097a22c007c44b2cd1ae3fd811311" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "note that \\( \\rho^{l - 1/2} = \\rho^{(l + 1/2) - 1} \\), and since the laguerre polynomial is orthogonal with weight \\( \\rho^{l", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793114483-pWvfeqEEz61U6reViW76", "proof_sha256": "ee93c6133cdb12f7b0e5d4394e8e0f253ed6ecfbae7232a7de03a7f16fbf85e9" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "yes, so in mol/m³, 1.09069 × 10", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793114592-nrgTaxlP44rgGTfGnJMu", "proof_sha256": "093abd6d0da36bb7615a48b36012ca8754ab2f523ee5cafd3587f3e10ae1771e" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "wait, f(x)=x³ -3x. the derivative is 3x²", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793114699-kgkQq13OkqcWAA8vfQcr", "proof_sha256": "47346e3cdb6b574cbcd34452bcfcd1447624421917e71f9d81b9ea2b1281223f" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "but", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793114802-rJFA0UxXZsUdoCGaju1p", "proof_sha256": "febb5622d8f983c2e8427f86008f12f5a81c12bd80fa3b271cae9e68dfbb681c" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "so, i think its correct", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793114903-53ybo3q733d1Ni0DCJkl", "proof_sha256": "f4b5aad396e065298bac26d95295b4d8984f7b746930ad0b89e009260484f1dd" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "so, to count the number of valid strings, i can first place the 5 as in the 15 positions, but with the", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793115005-RVra0iY90tqkpoyoR2u9", "proof_sha256": "d5c007f187a0668c08e1dea07026b493f7bc548384934d855e5c86f9fb33ca18" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "i think for such problems, the standard gibbs free energy is given by", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793115112-pSzw82XLPlHn53gCrGuX", "proof_sha256": "d83ae5c76a864b00f90ae6aeb8d8ff5b3e563b1a034fc063c15a00cc4c9b9ee3" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "approximately 1.0855 1.8753 ≈ 1", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793115220-HUz3kZriPWXDT116ySBK", "proof_sha256": "3c00f48317123fb2a9c4f7cb4db5370808f206a98cd5304a3d757eac769d4187" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "better: 0.6636 1.1785 = 0.6636 (1.2 - 0.0215) = 0", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793115346-mTgnsOeSgA5W4Gnki03R", "proof_sha256": "084251e09b0b0e56083213ba6a8c6f7e5b1b11df75ac8a288d8b307e2de456cd" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "this is at constant pressure, but for liquid, since its nearly incompressible, the enthalpy at constant pressure is approximately the same as at saturation, but when we go from saturation temperature down, at constant pressure, for liquid, the enthalpy decreases with temperature, and since cp is defined as (∂h/∂t)p", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793115452-j4aFABDVQMaueX6qDIMv", "proof_sha256": "d04647a85f83ef99906a4b7113029ba73eab94c152a60e734fa72b0e63c2969f" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "so n cp = 0.08 175 / 6 = (8", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793115547-Ww3AjlzjqCnWfPDnJIMx", "proof_sha256": "771afef81892b0e1e18e033381b89eb0e96b482b22534fd0149884401e01ccab" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "so total reflected power pr = |\\gamma\\perp|^2 p0 + |\\gamma\\parallel|^2 p0 = p0 ( |\\gamma\\perp|^2 + |\\gamma\\parallel", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793115659-uB2YH2AelbCUUcnAiIwm", "proof_sha256": "cec2488e06408270b8ce705b7c601e4758dc95f90e14ca2891c40aa6100f2bbd" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "but the work done by the gas is not just the work to compress the spring and push against atmospheric pressure. wait, the gas is expanding, so its doing work against the external pressure (atmospheric", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793115782-OJXQwJr0xC17o2HYkwrO", "proof_sha256": "4b93d9ad022d0f74e9fd5cd418c8a91c515c0d3cb24c986b4ac571184f5a764a" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "the answer", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793115886-fwqfEud2xgEx8LKbitrB", "proof_sha256": "96f7504d7fb7b9d1cce9609b6b0e9a8c2480580c75cdb4485f12ae80a438e874" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "the viscous shear", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793116016-ZoZJS87jSd3WnWO9AZJb", "proof_sha256": "956af6477f5776a3fdb73347eb6d74d77efc368b56a7d951b66b462e4f91efcf" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "after adding fill, total stress at silt surface", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793116116-rQIJZX9fDSYRG04P2feE", "proof_sha256": "c8d4e87fd83eadc1bdd4b0e66cb565c211a6b8f09ad10b59816e55724e94eb51" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "first, total energy: power 500 mw = 5e8", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793116221-egZZvN5Bx9DAEFV5dJVj", "proof_sha256": "c6ae27d657a0db6ff3ed0762fc328b1e1bc3c16cbec6e07705b59ebf44e22565" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "998.2 1.0036e-6 = 998.2 0.0000010036 = lets see, 10000.0000010036 = 0.0010036, but 998.2 is 1.8 less, so approximately 0.0010036 - 1.80.0000010036 ≈ 0.0010036 - 0.000001", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793116355-TR33L9BdUv0R9Q39IPIy", "proof_sha256": "1d8e3c804141df7fbd2ba4b023d4eab0b11312c1924c8b924466142785c0fce7" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "but from (1), m \\frac", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793116492-bohZyJZsBEUnVwdZa3ra", "proof_sha256": "dd4650ec730d522fa6fc4ce74e96ab1dbd478a977b4cc3d1a376baa5b16330c2" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "typically, in hall effect measurements, the thickness is", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793116612-plY8wAKLD5ALXxxlqwGO", "proof_sha256": "a6699d53fa2fa59a428355cdc6fe27e2add91dc87dcd997eecbdb405cb40ba2e" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793116751-JXtwBkSdRyLNY1Iwgnjy", "proof_sha256": "38f2bd32226dd41933d3d0eed550f7eab41a323290c22477ddf62fb62b8b80ed" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793116865-6UEXleuvP7xIiW7lrs5I", "proof_sha256": "a2d9bb515289b658d7b74b5957202de744d5b0c6fb422f2495f5a4e8d12a6f7e" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793116936-q5KThyeoYOWU2CnykBJ3", "proof_sha256": "d47ad811cedbb23be1518617e0a8120d26912ae9fcfbcfabbbc2604277e92f6d" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "wait, perhaps the answer is b. meningococcal colonization. if the question is using colonization to mean infection, which is incorrect, but maybe in the context of the options, thats the intended answer. alternatively, maybe the answer is d. subdural empyema. but im not sure. given the high csf wbc with neutroph", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793116978-U0HW9BH2NDVawHHlRzai", "proof_sha256": "498d1846335261ce98b1b89c9deadf96d2ffd8990537aa5dd8f7227bf16eb8cf" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "i found a better way: in some sources, virchows node is", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793117080-5E3noTzswH3mISEm4vXF", "proof_sha256": "c46c458ebcb8b010e9d08ca57713fd01409c671bcc2c4551a59dc48bc7c76795" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793117196-jOiH86JKWE4Qo1H7YnRE", "proof_sha256": "d5f42b692cee234a3e3f93456b9e442e1e9ea047df8da1f2f8b18580b14f9232" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793117293-L3IeKKy5gzshONhIvmei", "proof_sha256": "11a19f7e427317a7f076c1f8c4ba4ae7afe0f9758cf6d332edf52f18e53f6744" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "but then my calculation", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793117373-TjuDysxK9EYWmQgTaCXX", "proof_sha256": "e14884002d10cfb55f8a628cf6e0bbc8f6804c96d66580e4ee0e8cb615bd934d" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "x =", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793117496-icVObqYOV1SCq3NjOQhC", "proof_sha256": "4c4776bada9f00abda89c20f8d7b61f951dbf64839e95b155ca19adfbae38daa" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "but since we have", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793117609-HrYvR1GGDVPKuMvTvzSe", "proof_sha256": "c27c3b7ee9ee3baca620a244e44e4162adb8e983366f9256e7f829467a323a5c" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793117718-WwGvu7YXyNcfNSXSl3S2", "proof_sha256": "1bc7e028672516973cb4ea4757b22422f732e9490d9dbceb484c527db8ec6e69" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "then 1.806111 0.006111 ≈", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793117793-NYmaguDhhEc4u1vJQEWL", "proof_sha256": "adb7b902c5b7187f3cbe360f7bd766db1be9219b92fd1c60b7ca4c760cb4c4d1" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793117908-pKCeH7wTtw4d1hXuJoMX", "proof_sha256": "0779ff1aa214469dcdbbd071cca29f89a40ef292d7e231609889fab25acee79e" }, { "model": "nousresearch/hermes-4-405b", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793118005-OpAzreFtvHkq0Iph4xxv", "proof_sha256": "794930313d3f4cbefe7139b8fdf804ba7e70f86b82b18d6cb3cf81d327325d57" }, { "model": "nousresearch/hermes-4-405b", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "another option is that it might not be admissible because the janitor", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793118058-BHbHQskha057V7lrID87", "proof_sha256": "5ffdd69d05f678c000789bfd0df5b251b35e0f7741f32750f0f968bbdfe0c31f" }, { "model": "nousresearch/hermes-4-405b", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793118167-A8aZcHQVgXjRTcSylvKk", "proof_sha256": "57953d6ade50417d9c8eabd84fbe2660e833874537f7e32461a02e673c79a3ee" }, { "model": "nousresearch/hermes-4-405b", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793118207-Pav830bgzJjTuSVuKFUu", "proof_sha256": "ee62fe5821cafcd0da35f5ff015419628827cc2252914a105749f6294532fb4a" }, { "model": "nousresearch/hermes-4-405b", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793118284-cse3r1dlkg5HPi0Xm5GQ", "proof_sha256": "d2c46af34825f1534e4950c6c4732b13c36791216964eea6ae7b3b6b53f05603" }, { "model": "nousresearch/hermes-4-405b", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793118358-S77uo3o92zan7W6vZyTE", "proof_sha256": "bb6cee08f54dd7cb418792e99f7ee72113a1eb298f7d21fdc3e793940459ab82" }, { "model": "nousresearch/hermes-4-405b", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793118470-ZZhdVH7dNFZ35pxKJBGu", "proof_sha256": "50aabf36d1408d9b3478d4a0663217a6b717f3c47a3ed30b3e34dfcec375dedf" }, { "model": "nousresearch/hermes-4-405b", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793118506-y5fMFfuoOOCPmA3arDaE", "proof_sha256": "1d429f838e88c5be22d28f24ec9160dbf315ac5b7a178d6ed915c457fd76062f" }, { "model": "nousresearch/hermes-4-405b", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793118554-dqIkANhLYlWU74lHrRwx", "proof_sha256": "1752335ba4e91eb94ab062a957190f2cdcdb7ec3949c39b750bd123777afc768" }, { "model": "nousresearch/hermes-4-405b", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793118588-BB0e2KVJYCAmKAVT71Iw", "proof_sha256": "f4502bc3a88274eb314ef1943683510349e4fbc1f44c5c3310d9da11f42b16f5" }, { "model": "nousresearch/hermes-4-405b", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793118662-2Tzz61kdfOkvVWr4JreH", "proof_sha256": "7e22db9fccac1d28496c8c262583b91f38d3f068f39fe1edf93317bdd6291063" }, { "model": "nousresearch/hermes-4-405b", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793118848-ogX7MraGD4udzqAYAJXf", "proof_sha256": "9c3c3422c01c23c32f2fb2f2f90f3a566f1307037273509d8b2e942a465677c3" }, { "model": "nousresearch/hermes-4-405b", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793119176-zZC2wq0atYg6hRzv9LWR", "proof_sha256": "30fd6a8a85d861a3a4ba565c2d44640a03d35766fef322988ba704136ebf967e" }, { "model": "nousresearch/hermes-4-405b", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793119207-TvxkXyTfQrnIx5xYKlXz", "proof_sha256": "3f4163c4ea37cd6b98fb2efab8ae076663af00763b8e9a2b1b95d3e4d546f3e0" }, { "model": "nousresearch/hermes-4-405b", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793120845-bA0kaSMP1RQFBfUdUbZo", "proof_sha256": "b3080fabf065b195fedfad604d9b9a3d0be1a46d2e0839eddf562d65753a29a6" }, { "model": "nousresearch/hermes-4-405b", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793120931-oO019oPEMLwiSgP2uRkw", "proof_sha256": "26bc4e6df40219a251fa7c7d6271d49c5d9896d101d1c55fe895fc75b67e680c" }, { "model": "nousresearch/hermes-4-405b", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "89 / 42 ≈ 2.119, 100=211.9, and g is", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793120999-HXt183hdRGDtcht0a3WU", "proof_sha256": "288732f2483f1bc74a252c7a0cab78a64f90b4eddc733314d5c1306e4e241d17" }, { "model": "nousresearch/hermes-4-405b", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793121098-zxdpsxeLLkSvDPsDZC25", "proof_sha256": "73505989143cd6c3e76af9510d949ab58ee745bc530d83155c10a9a9efee2ae4" }, { "model": "nousresearch/hermes-4-405b", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793121148-M7jSecHd7cymyLI9D4r4", "proof_sha256": "939ffb57427116c4294f4cab3a836c20dcd55cfef909300eda02d3e6e725daf4" }, { "model": "nousresearch/hermes-4-405b", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793121207-w9yK9s90VdoOiNLybKIk", "proof_sha256": "38b94929716669cecf4f38047ac8453c925b164d24319f39f5d75b2d5bfeb950" }, { "model": "nousresearch/hermes-4-405b", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793121254-xDHZvuz1FVR1e5aUvXjD", "proof_sha256": "54043a66fad0b8a0988ebbad6fddab3c61bca1b81ea3a0502211a01049413995" }, { "model": "nousresearch/hermes-4-405b", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793121304-JKGerymTjYednlnoMi0w", "proof_sha256": "e8d66cc79cdc11fbf55be19b48fb62f850cfe316f931191f37b468330d2faa74" }, { "model": "nousresearch/hermes-4-405b", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "now, the inner sum for each", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793121338-g7WHcbs26OK5Z6xIODmk", "proof_sha256": "3ddb5d4ea373b76ed6bb95fd498a269d118403cac2f24f9545e198577b3c0ad1" }, { "model": "nousresearch/hermes-4-405b", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793121443-F4KhVmhGDqTK7yagceqW", "proof_sha256": "b3016af974e9fce8858fa43d349ba9b024808a4587e39bca90907d7eef2b3e09" }, { "model": "nousresearch/hermes-4-405b", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "ρ =", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793121497-v2qHkUQqkmtIjWeG8aYB", "proof_sha256": "9ae9016e3b9501f35910224bc7274bd1b94144bf3230d674b3b197d0b059439a" }, { "model": "nousresearch/hermes-4-405b", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "gold has atomic mass 197", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793121610-FHscZ93705bMvyYnVBHf", "proof_sha256": "4cae1abecd0c3821a01922cd20b297aef039a87a16600b4041c8f1cc60df4187" }, { "model": "nousresearch/hermes-4-405b", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793121706-dQKfQvPClYSXkfz4oc5j", "proof_sha256": "7e2c39df4f283569f12a38c5dc0ebd4408947fd46a45235d445c9d4017416874" }, { "model": "nousresearch/hermes-4-405b", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "but in the reaction, its liquid, so -", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793121799-R33K5y7KrSUfygmB6PYd", "proof_sha256": "4d571213a68d0041a8cdee2d49a0d546cd3d0d847e6ecf3c257388cbb70be564" }, { "model": "nousresearch/hermes-4-405b", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793121907-L3YYa4i7sncjrKZOXL3D", "proof_sha256": "35a9360ff9b9e6cf1fb4464223a7d3f494a587196e9a6c9e279fde15c195d36f" }, { "model": "nousresearch/hermes-4-405b", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793121951-RwUEi69EtWihTiak8pML", "proof_sha256": "d1ada630dd4931442b6ce054ab9f410660371073381b86a60371b96bb94a2d38" }, { "model": "nousresearch/hermes-4-405b", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "similarly, fc20 = c / a = 3e8 / 0.02286 ≈", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261027T120517Z", "generation_id": "gen-1793121991-nuzDTQskFnH3vzgCjkah", "proof_sha256": "8f0a2486c2a1a2ce7116f88b2f1eca314661ab8e5ec94178cd04a6329662099f" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793097505-yjqCGmEbp9UpAXTKcOYP", "proof_sha256": "2720cb5b91141c36be5e6d65509b8de3e310f706de1eee8853f5ad271daf2463" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793097520-BHGpBI6gdw1odjUFLEqJ", "proof_sha256": "536c65e6c2bb48e0fb5329c3b1b665a86a455ee9c457f87abb232c977ed48e57" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793097537-huGZBcHU5R2uMR4U3O5g", "proof_sha256": "f326b9bf957f414917353f4041472a09ae1eb2bc053924f224cacc701fb36999" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793097550-2Dp6lF0A52VfCKKYAIpE", "proof_sha256": "0eac212400ff01ab8d86f8c370e2388b590e6a74592dc8e4639e71b23b99d7ff" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793097583-2wQakgaFBFcc1DvMXMDK", "proof_sha256": "624c31a346c9020335c6a8ccccff1968895deb80e62a6e98215b1859c3f7feab" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793097602-kPwliQSPaYcRzRFGKxn0", "proof_sha256": "0cbfd37e333d6e8d22ae54fe329ca4a04765b60c7d6befcfcce7de6a30ad333d" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793097618-T7DCypjgJ5YF81szZ8KT", "proof_sha256": "1926e6d30b95cd1ab136e0c238190ad79dd4d75577b39367aa39b8a58955d48e" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793097627-VPTi8chWPxnwafDNGsk5", "proof_sha256": "45ad6824cfec7cca4b696a22225f383938da804b5710f3f42cf702632c4470cf" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793097639-ME5PZNvIh1S5PtF1M8Ob", "proof_sha256": "416d84eacca752f46695370435350cb628751a308e927b7a01cc7da54583567c" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793097651-HDqjYJ5TaL8xsgKOI4mf", "proof_sha256": "d6b38a29b7ba429b8fb4e5a48aa1367af790ffbef8864f0193bdb364bfe60d3c" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793097676-e1lVgymdpFEQm5OT9WDP", "proof_sha256": "d7b6b02d618acd2d865f94f51cf6f05fda091285cc32e7ad866a76392376da7d" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793097687-9SSvXmvnYAYMRUZmX6SX", "proof_sha256": "7cbe7479ff30f3b2c39388683ef45daf0ba8eca711d1e03197dccaa8a1da2380" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793097713-YBrnjtcjqOrRljQlTMM7", "proof_sha256": "a8470d764d91052be624abbc22e722384045244e15da5a9752249ed9132a0571" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793097741-Dx2CHENMxh3QnskTQb3l", "proof_sha256": "a0129bb7f94b16a4173503440b94d0fa60d74756f4704c1e568ad9cbc5f8616e" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793097764-aqzBWlgykkIow2zOSmwP", "proof_sha256": "3800c374b5778d68fa5a0aeaa82ce9564ce9a4115a90227deacc8fb749161a19" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793097790-UgfhfnnnTNqhBZ4tH7X8", "proof_sha256": "0c6847ba6f6cbe70637366c7deb3686e666a4a67f6091e042c522450649efa53" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793097816-DtUEWOqfHYc8PcW4cDNU", "proof_sha256": "48c71ccef33e5acdef4f8a876c645f77f2b1bb32bcfcfc8117fb8e1284e13834" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793097852-GRY9ZOPC4lGFp7p7TTG8", "proof_sha256": "f7fda831076ce6f89ba3198d6c38770941b25a3534dde1807a19ee8da449e44c" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793097872-MBJkvziN71p2LYpw2YOO", "proof_sha256": "6352822bac1978760046f1b54308ec035bff81ff9ce002853d583c837843f9fc" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793097916-ZIbFN7gTN5H25CZiE2yT", "proof_sha256": "82911857a4b2c311ae1a414ba7a4f2f8616fa7ff8d804a91025a1a7282448d64" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793097959-AyIKPgoZQLd0fjV0MhfK", "proof_sha256": "3d63cc26f7c4d53175eddc0616f9596f90877fbbbd7a2367eba560dfef3c9913" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793097976-G0b6Qd64gdwg61G7rPnB", "proof_sha256": "619770bca3c751915f3b3b75ffbb1d1ea3ff4f530a2cc2d8319549f50937e3b0" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793098021-ZmcCqYQ1QpbM7y4wP1L6", "proof_sha256": "8b7ff48cc976d2a481363baf3f2bd6d9bb016e50661b7263af8f2223841dc91a" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793098070-z7mHgPn6CjW2tGRtoGt4", "proof_sha256": "cc0948bc5cb5de1b74b0ed01cf776f2f3261554a7d232b29444546901c8c3576" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793098115-yw1fj6DMcv5vLsQPypH2", "proof_sha256": "5fbc6532c05159dbbad632e988c8a5582e34794681bd33c850a0b9e41827be41" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793098160-Jut7R3r4euul5jJJNJb8", "proof_sha256": "69e90da13e6eeb9881b95436e03f30f64a5256f168c41d251de69ee25e36ec76" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793099376-arJVTFsSpM8Ok6UaXZOc", "proof_sha256": "13b7e721a930b9481e21f36b3f7d61a774ebc9b5b549720013fa03bf1feeb1f2" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793099421-f7mdGUxX8hkZXxspridc", "proof_sha256": "3f8b1527a34e40800fb9e67c618da6208315e31b9c7dc4ebd3263b726c4691fb" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793099467-Zhn8YiNbQQ3GSBJBVvq8", "proof_sha256": "f527a6437e615f0bb47cb9e13402bdb3bac5ab73e41d3670b61e0e1d9fad1e50" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793099512-C0uCSZGi08x4a6CGEos2", "proof_sha256": "81b9132b5e029766f50160614a6245f8a81c6339cc81260a2c9f2c473f277fec" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793099555-PyPQh1J5CuUMO8vYUkfc", "proof_sha256": "bbcc9891c1c9f9c78463d766d1db7df1684447a36490ce978f8fe7509db85c3b" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793099598-xsyaIcv0ZjflVNPHWhon", "proof_sha256": "eb02b9d5b8569f6c5c1f31a8c3f197e696dcfd4b176ee0d60112816833a7bf12" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793099640-Htlf7nCDIDmGBQ8Z3GoM", "proof_sha256": "d5197b7d1e0f6d22589f0bd34213d5f66a639136fc3f56825cd38c0b6f5528f6" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793099682-yXvL28xPl5aCyOsvx2v6", "proof_sha256": "962413f1b99572f4e9999181060c120a871c6a20d373aab57dcc5c5ec4ce36c8" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793099727-EROyvOv9xE9ZSUR2ff7C", "proof_sha256": "b73b74017e5ab1c395c7446769a4e2ad7f333271adbc7255c38af686d96dc45a" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793099769-zpnOuyki6gZ8LGRxGsMD", "proof_sha256": "8bb90972a156c2556db11ac11475f35f2aa7e2d3bca93c3aba6095a1a3485c51" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793099813-CqVYPl9S04bruG8ayJ1e", "proof_sha256": "a75f86aaff42353cfc6ab1de6ba3ef95e945c8aae77845f235e931d5d0d213d9" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793099856-OipKmJRDaCUNmrmk2JF3", "proof_sha256": "b9b2db77948580faa4c4c84bde5f6fb51a862d5f63d61b00b14571d57661cc72" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793099900-cSyx2RyP9qvyEqhY4PK7", "proof_sha256": "77eb402ee13644a6ae6148cc23799f533a49a40cb4aacbd8a232ed4a255f2868" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793099945-gyy1fH80fXvjdcEl0aQT", "proof_sha256": "ee71998a62fef11e5421f858a4f60d5a2e9c5e2f87f6a87fe1c0a1e59cb05623" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793099987-U0qapW5xL8MbDZK7CTFL", "proof_sha256": "53ed4c4c4be1e9e8197a02d64bd6b28da997cae6d671fb970b2c7d155b18a8bb" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793099999-7ynzwm7NNzuFkZ1OulTw", "proof_sha256": "b16243efd9edd0ac0ddc754e0f592427036240ba4cec41c5fe65583f5af90b27" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793100042-un2gZqoCoqCnX3VWzp5S", "proof_sha256": "fdebe4d1f06ac7fc2b7499e183b0e864834ec8882634f0f803321a1033b031e8" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793100088-UST9LLUvENOIyr4JvcWr", "proof_sha256": "2ffaefa0c4acb5789f049f9779efc06d06c783315a93a4c1aed907ec617026df" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793100134-ugYcLehom4Blw2d7SXXn", "proof_sha256": "05e454c5f07333844ab93177db9fb6a34a32c60beaf2a7a24707cbbf8b6f3e7f" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793100176-7ETjFyuOP4K8dnjUoBG3", "proof_sha256": "acae1ac38028b4e7fe8679607601043c6d5c7d9a49de1a6e6fbd2d27a9f8a289" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793100218-4HMTwyDFWNVzYgH7tevW", "proof_sha256": "d7b236dda723c8c5e9f336195b406815ddf6a4bfb468a376b21891edb39cb302" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793100262-EXy2P85XMpBP0GatVr8t", "proof_sha256": "fb9e1de106e48d1952c1f60185def7f5b38f55e775b49e529757747bc1539321" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793100306-mrXx8LaLR4PazzA2abZU", "proof_sha256": "791a843b8f3d39b038e417a87c754123d385bc19135928e52f7b2187efa9f56f" }, { "model": "anthropic/claude-fable-5.1", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793100353-jrsJiMRCHBKgsKsnALI1", "proof_sha256": "8ca4cb4d92104fec4ae39b7cbc2d3bc851ef0c69d3b8d297ba85bbcd46d3592d" }, { "model": "anthropic/claude-fable-5.1", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793100396-6gwcxkofUvtXNCGxiGxp", "proof_sha256": "e09ef1375a00ba7d210da56422da6a4dd9d851d130c21edcda0a80ceb4455f3e" }, { "model": "anthropic/claude-fable-5.1", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793100439-rFQHPzlM2IkD9DmMCRND", "proof_sha256": "1cdefab2f7a0d7625b22683abe4fca350d4637652a678d38116c136217aa5a67" }, { "model": "anthropic/claude-fable-5.1", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793100460-GKtEtLM3JGzznA3aD8W4", "proof_sha256": "0fa2a5170959e345d21b6065a0b9a6d1827e986c90c52a0c3b6a3d5db519d973" }, { "model": "anthropic/claude-fable-5.1", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793100471-ds6QwLH1wUbmUvBmBC29", "proof_sha256": "bef18746e311355192abb817d0e231900d04a596787592855aed3028910cce6b" }, { "model": "anthropic/claude-fable-5.1", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793100484-inTwL4GTSPaYPmdCpKtc", "proof_sha256": "f59aa2d1d0e3b9a3c5dba70617fe3ad86ba60b52a6c8e9085f9b5bebf1b274f5" }, { "model": "anthropic/claude-fable-5.1", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793100497-e3MpkjaXY2wVF9f7cQD5", "proof_sha256": "3725cc48f242d7a1d4f73be9b5db3f2088b336def5d2501afb34d771ba745040" }, { "model": "anthropic/claude-fable-5.1", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793100519-pEU80aZRPOT3lUFnp43F", "proof_sha256": "0bf64e274596dc3adaab5ffb1e2c155beac586754f276ab1af2548df052ea9e3" }, { "model": "anthropic/claude-fable-5.1", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793100530-lRFNG5iPc6sgx6OfOA88", "proof_sha256": "2ca481e33ee9567e070326e9f06d7788be826ba0131688e33797ebe12c7bfe1d" }, { "model": "anthropic/claude-fable-5.1", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793100550-izw7J7VR4PivNdfdwOO6", "proof_sha256": "2823e1f8c2e86345c9a389e7adcc5d261c833f6b75dcbe080df502c99bd75736" }, { "model": "anthropic/claude-fable-5.1", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793100571-L1ORzpyfXwab7NxqatfK", "proof_sha256": "e76d632a97c38756b5a953c5a1f68d24225ff3017ae78efcaf213faf2ad121b0" }, { "model": "anthropic/claude-fable-5.1", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793100583-Hk46tgOeY9R8Ko4xpcVk", "proof_sha256": "333e5628a03b394863cc75ef2ba110ffc49b53b9ec62e36f354477e3987e4486" }, { "model": "anthropic/claude-fable-5.1", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793100594-9R0jYmrBzYlis3RYnXT4", "proof_sha256": "8fced39f60fcb9f8ef5b59b16e200d97bbe0171b249cbcee80d6096cdd9b1408" }, { "model": "anthropic/claude-fable-5.1", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793100614-5AdSaBFYiDAKOvKdgWm9", "proof_sha256": "b6ed10d13ef206124995b2dc0ff35d1d0392316e8745bc96c7a292c8f3ba3dcc" }, { "model": "anthropic/claude-fable-5.1", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793100624-qlWXisKTJozSmgtfvKT7", "proof_sha256": "f6f27f52fd3ab32ecb66d71346b80a38fc9e0cde7e9bace3e0410db15d8fc589" }, { "model": "anthropic/claude-fable-5.1", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793100635-TqycbaCn3erzsqWxPaKJ", "proof_sha256": "07d3551e1a11793b03c0ee035798dcadbac341834048a772a43f57596a7823cf" }, { "model": "anthropic/claude-fable-5.1", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793100653-aZa23zWuBGUDGhdnsxfF", "proof_sha256": "179b90564913376d82167e6c94878ed1c03cbaacffa4f1e67352a4456b61ba0f" }, { "model": "anthropic/claude-fable-5.1", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793100673-XAAmNw6ZZgRU5j5kXaRD", "proof_sha256": "b7408f8cce58164cb17ade60e86fa17bfd60073a77c6897573134b36b814b966" }, { "model": "anthropic/claude-fable-5.1", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793100683-G6z97hWB8uCczX9357ui", "proof_sha256": "d7f4ee89e0505b6edf91c842d88fb8c810c49704145155c71d87571018ffa8bb" }, { "model": "anthropic/claude-fable-5.1", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793100695-tZGMtvEQeoZbe9ErZrzl", "proof_sha256": "aff694e9713e7d61332035eefdcdca201142c759fb3f92e2373864a1bc9e4f8b" }, { "model": "anthropic/claude-fable-5.1", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793100705-VKy5c2q72a37RIT4dk1d", "proof_sha256": "cd8cff29e16332213c02d2bbcbf32692350905a32449617ade3f96c286ebac67" }, { "model": "anthropic/claude-fable-5.1", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793100715-E7TrU8VAaKwq87AP69RU", "proof_sha256": "bd6341b35bdfbb5e2e3808527894aa9e5599a2e0911b0802583a58dca9dc6a64" }, { "model": "anthropic/claude-fable-5.1", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793100727-pbeDoPNP9pcK61kZSUVx", "proof_sha256": "c7c918c9fb63e1cd3651a7ad480d10470444a1117f555d206cc170848c707395" }, { "model": "anthropic/claude-fable-5.1", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793100737-vJzVsu0uB352kEiwB4iY", "proof_sha256": "046a71d9beaae85ccbe738f933f577accca878b1f7f65c3cfa4d944ae61f8119" }, { "model": "anthropic/claude-fable-5.1", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793100757-5B04lxhcp464p6yjVnxk", "proof_sha256": "e76f4fffced72213367d5665e7e596ff3c2e54b11d010f09f85af5198ffe6e11" }, { "model": "anthropic/claude-fable-5.1", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793100769-wIrw4PNFa9kuti7lVWsD", "proof_sha256": "735b1457c2bb7cd801fab670e7c8c22aae0685996d28d0c0e5e321ea3bb2898c" }, { "model": "anthropic/claude-fable-5.1", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793100781-CzvqZqShwhGLQz8YOvVd", "proof_sha256": "ad0a67855c8274f774b2170f4b401b395b4d3c9e78288ea9f3ea5f428d6afee1" }, { "model": "anthropic/claude-fable-5.1", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "content_filter", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793100790-dhISxHbJOcVnUZ9cDOro", "proof_sha256": "6435d589310ee26963bea87226cdd0f6258d8210509d4a1f56af056ca9a47107" }, { "model": "anthropic/claude-fable-5.1", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793100830-vHeYD9NZJJTQiunFcW0Q", "proof_sha256": "a90427cfb28992595c7a5b6af1852c8381906e9ade27fbebff3f5f2da038186d" }, { "model": "anthropic/claude-fable-5.1", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793100840-7J0x65ZvJjyuIRjb0wQw", "proof_sha256": "756cbecd19e6f5a894663a8e9d7432bdde96542c21d354d67331467c9bd3e7ae" }, { "model": "anthropic/claude-fable-5.1", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Anthropic", "run_id": "20261027T103825Z", "generation_id": "gen-1793100852-LmPTjT4hkXITZh3q2N8G", "proof_sha256": "f69b0c4b51302b323b2ddb44ab00e876b2425387c170c1ff7ebeec0582aa516f" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097505-1bQKEKMwbJ5DO3L9MEqJ", "proof_sha256": "546be89ddd1045894875efe9f38ef7f1ec5dbf6c024b85d6019a5cfa75be2b5b" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097518-OG6iX4UtsMYmwGdqiwnN", "proof_sha256": "a5fbe5e27f7f26eed7b4b73d24df14c4b225f20cb23dcde89e2a735891aeffd1" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097531-4RYEDW2r9oAxMyLwUQVV", "proof_sha256": "dcc9d179dec4986a973761d2a57e53af105b71544982177217d6fab094c21229" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097550-pZjzyXVm9Ge772k9ZxOs", "proof_sha256": "b07dfb07658a8ac31db3c79288546d423a7c0f91730f7c20eb17c3fe8d4823b4" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097565-qT8gPscbSSpDMwrwK8zd", "proof_sha256": "ed10933811f84f28ad46c909aa3421f60d7b344e77a966e741393dcb8968bebb" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097596-WXijkOKkQNwDd4Ssva3F", "proof_sha256": "5c7d4b9d3c4486268da4953228458c34bae9d728f675894990384215b26612e3" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097619-OC91KItit8xrcW1T3UMN", "proof_sha256": "5f48bd019df213202b0ab5cd270a3e292668743b0972cbe2ceb601863eed52ec" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097639-7PVnISUoILl05ooIE9dO", "proof_sha256": "1fbe3125cee04accabfcbccf9de5ee9ca02a60981c3f078f343652997721f147" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097651-txAI2JSurGIUkBeY3ZeW", "proof_sha256": "d35b0fae4264c037a0c619744ccdbb6c43549f63a194f4c04b58a15ef5fef836" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097671-ImqSyXFFUGKhK3hUQtXd", "proof_sha256": "8a044e03d54a430d06068a15f0e80156c5e2cc3cfdc09c5cb8dfbf6b9afddcf8" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097687-dCrk8wenq6NDLtM8Cuso", "proof_sha256": "6dac01d4019e6fe88b4846d816dd9291e7c1930da0d94723b61270ba5d3a7d4b" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097696-jSE1m2uCnDOg4WkDN3kH", "proof_sha256": "61b64a5827b514bbdcc4ac62331e28050bce9f06a932ba27d83f6edfc54d46f8" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097729-QmiMiUazp6BO6Og1de0J", "proof_sha256": "2f984e79e5727af1970d333e607849eff21dbc4ac2567415926dfe78e06326d3" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097748-dVGKkFeDWxXvSOy7COcS", "proof_sha256": "473c189bc16630e63043e2a061dd814d89a8cd163b74fea22dde86e182d097ce" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097770-zAEPPo1qigRby3pJrStK", "proof_sha256": "3b00f37a224325675ffbd41111066b7c9e425162248052ff8691ecd575d183dc" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097797-qkrvxFDeKuVV2h67YOy6", "proof_sha256": "8cb8f8ea7b16cc4157bf8b5a2c404c99d23d2c6b4c2ea2dde70a2feebaa44d2f" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097818-XPSUe61JJ7cyvYeODkjp", "proof_sha256": "fc81dacee1a28ce55889b5a76583ad57fa274406e072adf40992e44b0984bda9" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097851-rdbhxRkpdmbEGxQC0sqU", "proof_sha256": "ebe6c5a5ea02e2f69b50252e2176301d06993ab87ef5f912595b36a66c50e582" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097875-niBP0fe06jtoqFDnScbP", "proof_sha256": "ac0ccacf3e79206d59370c3df9e25928a6d2b25fdfe4a7a9f600eb0c72b25832" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097918-4ZYvZ569Hx59RL5fLcVH", "proof_sha256": "19faa33a2301f71175ec0cc70dd56ae70839d9681955fa83224666be808778eb" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097960-oGLYHu4saRPLnVDZGNUh", "proof_sha256": "2be50b87be30134160c5475b49549f0a67a0e003947a7193d86218a6147aae2a" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793098002-8yOPQIwHAGJGIxXeGUPo", "proof_sha256": "d149c7c2de43070dc62943b911e7e0551af81ee74506278ed5ba41198dd2b160" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793098047-OaKS7ZFKCrm66cv2hCIa", "proof_sha256": "84dce42974f77fc225c5da3775e66f97ef008855d58d82cdd7946418c31493d2" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793098131-eqTfxtigsAXvGDy9NClJ", "proof_sha256": "d42ea8d1e551b454f53eb6c7ff6c0df9f43ccaf4232ef161cbbfcfe8089deae0" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099376-VpydUeHDCq9AjNusng2n", "proof_sha256": "da821f8717bec2cc57a2510bcee19de24491ac7c5aabdf1703111325ed564d23" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099425-hKi6n5jxesI1GBa4fc9g", "proof_sha256": "f983bd2ccd704cd5b6ff58c36957a117f072cce03bc966e0454a7a592c02dd76" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099471-M1iq521vsQwUtZmMorLd", "proof_sha256": "68538b825e64ab4d84f0c0979c4705a637c8719213031d9d3402eb5486161243" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099515-9kwiEu9qvIZXjUYSaHTt", "proof_sha256": "08b2d977ddaca54b95fdfec16d9ca28a8512bfd1fe8d731c7517e6dd75059faf" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099562-SNEs0yHJiw38JRsggcyh", "proof_sha256": "a4c47c38c9c989f1ef0a1a7a1ec72d1513f4977c6890c73d8c72df6ff389663e" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099608-2witPiqaKxjQ5G45I2vS", "proof_sha256": "5037e4bae8d198949f38b9000aa811309652195980eab43962a020506144f481" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099654-XD0DH5Yi6dGnCFc1h0jm", "proof_sha256": "a0b17ec71873994b6d98565b0bcdda39ab80eafd117904d434253ef5c6b4ec8c" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099705-Fw9rffMj4c4hSsoxRhSF", "proof_sha256": "f83c79f6eccd1e99b31d410c0bd5499631c4963de1e5814aef6b5399dfa106f5" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099753-81SJlh6WNky0aLTm209T", "proof_sha256": "59e20ae63f6c24318a31a2b4d53ca01dd06dd2ca9b51e0b0804b390716298710" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099800-gLqvgydjFhw4Z3ZTNm0Q", "proof_sha256": "93b634f39ce5eb88e7d6b15f9f6a38ccf9c2682a4c9c104f8553396e517999ec" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099851-BDxAoHiih7V8HiVzSkGD", "proof_sha256": "10fba8f2ee90f3e28c5c575b708c12846bf3dae20aac89e85991bd4c3c184e6e" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099895-Law18frwoA4t0BiBiuGr", "proof_sha256": "8f200e6407637218657c1425bc227c1d8efb96ec30e2a10b7b834cb337a76322" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099938-Vga8GkIWEgYKF2tkuEId", "proof_sha256": "0d2a79ba3691bc9e203c9ec10f274055e5fcf3c48abf57692dc2d09f92c1835f" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099980-Xx3zTcEs8yN3VGhiFWps", "proof_sha256": "028d5b67a421ba24c887f0bde6512f44a81f266504ae033835f913f7c27acd26" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100023-XGtfW0YQsNxxWwCuvThn", "proof_sha256": "0249b034746e4fc62770634febe2103762fb52b7545b608741dc2f5863752a1b" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100066-INDzub9yf3QeS9glTg3S", "proof_sha256": "525f63b556228c331b60815699b00b16af08ad7c47d8bc799edbeda36f072adc" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100100-G6RJDIC87BODPa7dqBhp", "proof_sha256": "eb5598f50ad506656ebf29d46ce82a715af8c5653083a53a0ab273996e0fa5e3" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100141-JdHEOX29zUgW27QNFb8w", "proof_sha256": "cc20b07dc42e35930e302a327fcb3bf94a33ff4802b9e67503cc88e959c18371" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100185-EEF8XM2wOLymzSiCfdvj", "proof_sha256": "b93ef3b2386cca9d47dcb3fed214b4079776d83fa75658f9dfa225a66f9f6fbc" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100213-i2TODGnjFSzGVwR80ytC", "proof_sha256": "fcae503e678071fe3cc3e21a2a83ec417b5d40c4c2fbd649ffb180b094c28276" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100262-CI5suliqyIyyVCKklWLq", "proof_sha256": "896cbdb63b203737132903ad665f4e84125ff9f31b7114256554707be7f8015f" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100303-dXtK1qhC51EybbAsccf6", "proof_sha256": "dc1f5174188b922e8887af338cd7876f7707d8c57df5119a622e2e16f8b708c6" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100347-eybktPgjiWW2abD7QHq9", "proof_sha256": "b3981dc8696232fe2062add9a2df09cc9db0d5dff4c208e33c26ffa3d1c59cb6" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100389-5xNTPFzLBUdoj2hnAY2g", "proof_sha256": "9c91e19ab2e9c345412d743058e1c555e0d900807f38c2ce326bd125eba560f3" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100438-r9LenDaPOoAXIDLbaeLu", "proof_sha256": "4ed55b49efb4d36f5cb92034f87002546c1eb11b4a94c1b8736e671fc4c2a486" }, { "model": "openai/gpt-6-astra", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100463-mG4yMGAU2AwBB2ZSJfSB", "proof_sha256": "80633381a3e6824db9a4ce7572fbbc0aefc8e0af84af753c69afb5982a1fa2ce" }, { "model": "openai/gpt-6-astra", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100486-ZzxWvRHrKsSimRtbFXKy", "proof_sha256": "2eab8ff8bbe2e6b6856c16444cf507338b58fc0967d7236e8d51988a515a0b93" }, { "model": "openai/gpt-6-astra", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100514-phkiwNByvXjTPRXZkv6G", "proof_sha256": "da5cfa9c8f23748aa46789a8170f2fa452ac71e232f0552a9cf4c64121692fcf" }, { "model": "openai/gpt-6-astra", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100527-DV5fXM5pTvGwof7xOblX", "proof_sha256": "a64cce146260c136e94109a9cdc5fb62da9df61b3538000beda954b058e0c64c" }, { "model": "openai/gpt-6-astra", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100547-yZKNWHogXKSDyAP6oEqH", "proof_sha256": "e9463813e8343f20f464ab9322cf0da82b2b46c7ea0a06f0fc1a3ed37fd2bd7f" }, { "model": "openai/gpt-6-astra", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100568-K7SdVEMAnTQp0BHX02L4", "proof_sha256": "7611f1f8c1c280aa08b868f6ed27d65ec9113111394d5bdc0e2a820a9b755cb5" }, { "model": "openai/gpt-6-astra", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100582-O1Z5igo6L6Rs25O0k873", "proof_sha256": "5070da5cfb07c27edf207c5adb9a52143003fdadeab385c9b1f6ca1c690715e7" }, { "model": "openai/gpt-6-astra", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100593-mAaG0MZQUoUtnNgWCU2C", "proof_sha256": "c331a82170a1aec0f9c26018f9c57c336dff044ae5b47e61716168f21c3de1fa" }, { "model": "openai/gpt-6-astra", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100614-FsQq2UyEx8OO3oIVtPUW", "proof_sha256": "5cff8d99adbc57dc79e4020db3c7f3276838216aa2f5d2a730d969ddcd388f45" }, { "model": "openai/gpt-6-astra", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100624-tcwzR1nGOMLvcQMDhbwz", "proof_sha256": "c41add053d863940d248a28eda79758c980809578ea04818a7f6e07faade3c24" }, { "model": "openai/gpt-6-astra", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100635-Z9hY9FkA53fL3xEwsjvQ", "proof_sha256": "59d0de8d5b85f8ffb1b622d984d80947b8db97ecee9adce5510e37be8bb00098" }, { "model": "openai/gpt-6-astra", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100656-IKS0iLCCs3OcFFQhymFk", "proof_sha256": "5c50df9b3bbcff8f8f6286b1c18ac42d807bc3ab0606b16a12f34a08bcb1b6a9" }, { "model": "openai/gpt-6-astra", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100666-wCtKGozqSSBILTDQR09F", "proof_sha256": "26d7901f84ec06e3b463605ceb113a73b776d0ab564959d38f69c040afc666c6" }, { "model": "openai/gpt-6-astra", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100686-ocm49GrlYDGNgI8rDlD1", "proof_sha256": "447e6a3cbaf45e7425a95946bc29f164a5aa25d8948314cb0929c56d38000687" }, { "model": "openai/gpt-6-astra", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100696-UergANBKqH6gtiHfAEYL", "proof_sha256": "0df78acec9e99377b6f7fe6e11643187458deb028da38eefada7db7d5cce6a09" }, { "model": "openai/gpt-6-astra", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100707-SRCr6d8tbpBCLsmMCKAk", "proof_sha256": "b1a7b6a0273e91dd00cd411d1ce517105b0e3910277c3f33b77700526b3cf800" }, { "model": "openai/gpt-6-astra", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100719-enS0HoAfyHgc0G3X5Jm2", "proof_sha256": "2360c47fd17ccb3692e47270a7aa644d92344b7b73ef73a0d56337b465b63663" }, { "model": "openai/gpt-6-astra", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100729-AQ4qBCFJQM56SXC6n9T4", "proof_sha256": "f94bed8fdbba7d72bf46edf6dead1fa4c09762d77ab2d8056a3548b4b99c029b" }, { "model": "openai/gpt-6-astra", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100740-sT3RSK0LuQW1O4BA8yyX", "proof_sha256": "27eff5f3e16843fd62a818208789cd91f1c67c0eba3da7202e2424733922ea25" }, { "model": "openai/gpt-6-astra", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100753-erCWILmxXFtkfM5uXKxc", "proof_sha256": "ddc0ede1b28f7e0c6959735441742ca3a8c3fcc2096c714a056c3aca117ab2c8" }, { "model": "openai/gpt-6-astra", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100772-XSEioVgmz3z3riPMprnh", "proof_sha256": "1e659b6adc5427de1321b06e9416e442bd98eecd482bcf933d528f092f4ba3ad" }, { "model": "openai/gpt-6-astra", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100782-WdHHmJcu4X8rRwTcFQVg", "proof_sha256": "22db6dbc930dd09fc5d73b002219e758377c96838d8d66fa2059fcb81b427383" }, { "model": "openai/gpt-6-astra", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100802-pECFcPMzKad4WeMvdRo7", "proof_sha256": "0ff352ccc16168849b0dced701d739f11309b3eaf1c1be925a05c7fe2e51441f" }, { "model": "openai/gpt-6-astra", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100815-JDjxkcCDjU4Bh3hWxT1i", "proof_sha256": "33038a2e75764800b4dd082650bdc78adf52d97c42afde92fda2e09c80267b01" }, { "model": "openai/gpt-6-astra", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100824-vT0OLfR1k6mnJCmESN6N", "proof_sha256": "e8258890c8c95e0f19e62f71b9f279e6f3247d69cf849d62d3d9ad670395a43b" }, { "model": "openai/gpt-6-astra", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100836-xgXxh3ffDk2RsMyrrSia", "proof_sha256": "80e518922100bc628a4a059f09588608904368a61ac390348e3971b058a2968b" }, { "model": "openai/gpt-6-astra", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100849-aHaLHGdXI1ON2dtuoQ5u", "proof_sha256": "182f7c8625e417a28d458ee3c911793eaca2d81852cc21adbb5a7e464b0a887c" }, { "model": "openai/gpt-6-astra", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100859-NmqkUmlUCDg23B8lOE7p", "proof_sha256": "4e3cc326625c4678892a457e633fd5673bddf1aa3aa32b2ccdf8f769543ff2a6" }, { "model": "openai/gpt-6-astra", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100873-tvilB81ZgyiqNGz7khku", "proof_sha256": "377ff5d7c59c34f0583d32f81571416461e5bed3f130a10431f94cd8622e0dc8" }, { "model": "openai/gpt-6-astra", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100883-0c55cxNNCDWYKxTPAfkr", "proof_sha256": "82a5bdfe34d743d779bd8fb6cfdd52eb408f2db433dbb0140def27640ff44c42" }, { "model": "openai/gpt-6-astra", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100902-pRGOPeVJ84xaEKAZ4myx", "proof_sha256": "96bb496780bb48eb69f68e8e4c7aabeef9538ef870c221f371af90c5fde047e5" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793097505-ln7xzsSGEV10pSPIyW32", "proof_sha256": "b4b144d3a1c0dfdfa89d06a91564c16a40f8494da2b188a3c3029ceef66520ee" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793097519-k0sunkd7wKWJttbm1iEp", "proof_sha256": "763311860e35b2f094888b1207b707fe1db44bffb9019c3215829fd367a11046" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793097560-PtSFKvvXNmcWHeI0QPk5", "proof_sha256": "96e128dfa52d9b6b6b86dc80170e94f4dd8305524716df47e905ee95f1b6ffc1" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793097571-hLz5W4MWVabZL4BXNnCJ", "proof_sha256": "5cc8b0bc55de90aeb8003860bfa0b57741f23e4b00a070e881139a82a021f719" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793097596-sE8lt8853XCYs5IxN4wA", "proof_sha256": "9dbdf185c4a4fa46e24b9aa990ace7d9f391cdb5f542eafcfcf9cfa1ecb881e8" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793097624-mje3EXv8gfiCrthWBwFK", "proof_sha256": "eb0950c907d55afd20647fb3abf60cf0704fe54a097d3b4272c4207d36a809c0" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793097639-vLNSWEgMRmz4xMWV30SZ", "proof_sha256": "087a88d405536c0e26af3f8c48d0b1ee06723256f1e726e359b8b26872c88004" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793097650-1HCtdBXQ6MZ0cwuT5QwZ", "proof_sha256": "b653569b07def4fa2d3194e5fff85273d4a3bca80782237aae4def8f001c252b" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793097676-SwpQwQmnLZdPdbhPsbC9", "proof_sha256": "3a9eb67900d8d40c093ce9cf480a1c61de8911ecb6841f29d78f70a11a651608" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793097687-2KadxSNrgT1JKlxqVHhk", "proof_sha256": "4238cbb08c1c51438cc010db456e100a7af4e4ec092caade3387fb248db4217b" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793097716-XFUGRjVu9Ynt9tEYiV4g", "proof_sha256": "1962bcf35552f50e64e94a1f4245fad1c95374f7046aea9095298974d4e381a6" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793097729-hhQZOp4FRqPzF5hiWa2Z", "proof_sha256": "84a1f5adbc80dca781d7b3ec78f8071a07b0e56722563e36655a7518932271ed" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793097788-ts1BmGSJcAnrFsByY88I", "proof_sha256": "9532e2af25d8822ea2aea150d86659e69a600a3d55b7421d12870786658b4c0a" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793097808-GzWRk39yRTD1kGRFBEkW", "proof_sha256": "3c659fe15fb18853fa3f2a9f5602b9f60e2f5d69cb3422702c26b917dde1652d" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793097820-sKHyB5192EsuP0iQYaoO", "proof_sha256": "4ede5f18d74f89437c870494a15ad25146ad242760d75f3bba6f00c1c21b6cb0" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793097851-Hor3nkdIKdwtrX8e4iwn", "proof_sha256": "58ec5f681e47035d32daf1b31e844fd4a36a6ecaf8dc2177785ad8d89d3c1e7e" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793097875-squWGe5CyibS3lwQ1YGV", "proof_sha256": "258038d830528df1329fc2037402ac1e1403c2c980bf830245c4d813e2e98b2d" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793097923-6gHt5lfGEKC17B66POfs", "proof_sha256": "a46e26fe1d8cddcd063fa7ac348eca79b901936494aea655b8c15b006a94c79c" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793097966-x3TA77viZEKRKKMys36m", "proof_sha256": "5a619dcd107eef5c283766afd23f3e1b0850c0574f04985cfd61084b41c1880a" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793097978-Q24FhUsWEJNkse8yUiH3", "proof_sha256": "7ff43a97c2176cd0b2141e3befef5e8c63528fdd1e3e57f3460423f63b14be6f" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793098018-MkjIBxlKNuCJ7nRoxkcL", "proof_sha256": "255c5c15107c6c8946065e9c6ac6d71571dec10b2a6f0dab442e90b31e473339" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793098052-REBwBMYRsJHg7q89wZGc", "proof_sha256": "3b16223549f48a782146d25e5918d8ec9e6c46cf4962668b4b7d6423153e036e" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793098095-zC7LfjbOSOh6uaZEXnxv", "proof_sha256": "415e106b25ad19a8625883549454ac3a4da22d38ebfb569ca77ef8e77e9a0168" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793099376-TH49bv0mCVdMPw9e3CNv", "proof_sha256": "e61472cab5eeb0d6818393a49c360cda074c03f807579e8bb65d6d5075e326b9" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793099420-Qza2yX58ZFx8b8CmG6xg", "proof_sha256": "6353f08dfd3b1996a6cb28943d69a3ddce8bd1389b9a2a44fb8495ca40bdd1e6" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793099474-IT2joc1prQDYCG3E63Io", "proof_sha256": "520c2dc56c8788bed2f52fc617ab5601c08d5c589b8d51ecfc96a8fd8d2b53d2" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793099538-HzLi1nSMyxHzQGg0tsHv", "proof_sha256": "f7fe97e8eec2b2de2d7f6a3c7f76c12bbed90acd646bac5e3b95002b5b118ba5" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793099584-Nrni0htA4mtrQkamVcqL", "proof_sha256": "31326b7ed13ba86450d194811588066e3c4312ac6b330c791977801da8208c0f" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793099633-uFjDhvhvM5hh76iouqCm", "proof_sha256": "cc35965b6b97e8d929a5ce48d1f07317089280b94496f069a74d95e4a5f02a28" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793099714-3OPTxe9inOUCMJ1S9s0j", "proof_sha256": "67b5c656a5f2d75251163cd5a591662d3719e5203176598d27d5e4a97be0adc9" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793099770-gjyGUI7sN2g8XqcDUnKq", "proof_sha256": "923f5dc4a6ee1c37cde4ef5bcbf514081543db517c7313c400d51d0165ba6af4" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793099826-TxVuyL5VkIIds2EEhB3R", "proof_sha256": "f914a423e74691d9d46105390f446c28de98fbbd3485237eba12063531719c8c" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793099875-r2ELuIez9EhNjEvgFYfi", "proof_sha256": "4570d6531616d26af726f24d6a5615b69af5e46659e3a703b23f69c3ad8bdbf3" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793099918-jY8oBcx4Jq6v0OkFKJJ5", "proof_sha256": "d36e49a3c30225f7c254303b124fdcb1340e738ae929c797b19c81c2a6d0245c" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793099988-HBrDWfKJElF2rJDt5a4Y", "proof_sha256": "092025403fa74a2d7502c7eb7440a937a547a0b9ffe0e15ae3d429038b7c49ef" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100032-DfFehBPcOBJUxOVREIhe", "proof_sha256": "a73be03b50a4c3aafc16cbc8c7ef9d2eb0ba591fc9bc1bf5c000ae50c390708d" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100074-stUOAXPpNWveoHDVmeX5", "proof_sha256": "64edd30e7b421be314aca40daf63a0d295d9cd6f312e39077f8d4e0ff9235610" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100119-3q7cunGZMdKg6Cp84sdl", "proof_sha256": "4ee18fe2bc97d1e9cbcc878c54584fdc616617ceeff2d969729f56672a312341" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100161-wMN4Jty3ARTO8xpsBNkJ", "proof_sha256": "ac9e4798289ccb009ad2b609245b6fbf2fae9d29e96207f95b7f1cf33da6f19d" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100205-45YVvyGgIZ7VIsYOdd8a", "proof_sha256": "67aefe4321722b5877a919551258345edfdd7c26cd87d82892ea60fa820d97e2" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100250-IUuDQyTCYnzjP4oNgeg1", "proof_sha256": "4b9a79834b463fb08ade5e316fdb9846e36533ce39d339e144579d23a62f7ca0" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100299-Rau2H8kgf4lRleUirYOx", "proof_sha256": "8a5ffc5cf50616b872e28453811d4438eefa3a63bfda902db5fd8bd1f1554625" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100344-KWYO9qVysHwa4MYGGPRr", "proof_sha256": "811208950b65582933f3ed955dbe4f8b07b36c1f690a338e0a5bc6304ba7f147" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100389-NUZ7KAILicU6CloiiQSb", "proof_sha256": "3caec3b42d80f7950d3a5f1501525d3d5f5f0560281f622cc61cc8a9f5f56efd" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100452-Fyw5LqWHfjTgyE0QMQPR", "proof_sha256": "1a9ad1f81e40e7b11491c873d660f38c70c0e45adba0849e0eca89caf0f2536e" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100471-2KaCSCMWRFlbVh67gyx6", "proof_sha256": "4665ed70a7f0c150ec610be02518bbec0e2912d7618bfbe495f2b8e7836bd12f" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100483-zx2oR2yBC0sYtM5izyMZ", "proof_sha256": "eb3c445cb206b3cce0c7badc59e49ce0a046b50f98ca405388d3ac9922dd48a6" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100507-ZjhCYWe0Z3xvzv4mAQ9q", "proof_sha256": "88978bb0d26dc3630e140e71de1cbdd6a4d5fab6b63db028c611dd54a44c6a60" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100530-5HvaADfXo9A6tpGw4LTM", "proof_sha256": "5fdb0e4e826c4b485f416a88fcfb792433460bfb9f24e3eea9dc55979c251c6e" }, { "model": "meta/muse-wonder", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100565-hNtxeXaXMPPwG5ZpR4m8", "proof_sha256": "f74edabccc9d77abbe71058734c397cb10200c0f9b590bb3d61b7e369fa0cce4" }, { "model": "meta/muse-wonder", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100575-qMMUEId2Q7quc3YXxEuF", "proof_sha256": "b9bd4ce3aeb75547bf3073d1f41bd17a1bdfa1c6efb7038b5bd708dad26eb8e5" }, { "model": "meta/muse-wonder", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100590-e5RvQ4V1kqH9GNiPyKSc", "proof_sha256": "8990a82aeaad51ec2624e3a9b6e16f194447e2fa0a4460da475d8e08eeee9b8a" }, { "model": "meta/muse-wonder", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100603-1XsTrr1ovwDYOU21NbCF", "proof_sha256": "bb652f546c5ed41319138836287bd1988b679aef047a377fbebda5d7e69543c7" }, { "model": "meta/muse-wonder", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100618-rPke4FTqKEz85nP34PNM", "proof_sha256": "475c4117190407c3444b0700323a67c59e60d1fcb6ede534903947d77dcb43fc" }, { "model": "meta/muse-wonder", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100636-6PVEucSFrsKIML88BTtr", "proof_sha256": "69efc3dcc24933a5bacbaeb2d20cc692900626d3500ec1e9160cd72bdd4af87b" }, { "model": "meta/muse-wonder", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100658-Seo7qXXDPHUXS3zBXTVK", "proof_sha256": "4f5eb1ab749c54dc03a40bdb4c16caf3187917dfd101a938286c5fd9ca8a9c20" }, { "model": "meta/muse-wonder", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100690-dPZTqBojpSdyR0pj7ApD", "proof_sha256": "d42a747d2129ef7f7773a15818753b567764a7d7f42d808fa5d86d4157516774" }, { "model": "meta/muse-wonder", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100703-m8Jv2AwK2b6BLRDQs6ww", "proof_sha256": "295255f31fb862bdb6d6555600c1360d09f2250331100a52effbfffd14c2eee8" }, { "model": "meta/muse-wonder", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100712-9kLJcvSeOdnvjSImrq4L", "proof_sha256": "f4ecfb83b9b33163c967d68cb722d6a4683bf35327b6091e4a911416d9175227" }, { "model": "meta/muse-wonder", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100733-mZOoTxJg9kCURNurUI9I", "proof_sha256": "b7b37984b28288d86720c5076a465d75aa488b689e7dd14b0568d40698247afb" }, { "model": "meta/muse-wonder", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100743-BtSQcBcTjCs8SsSz6Wph", "proof_sha256": "9848e42a762b04b4854b0f9be3dc044b2af2125876f8d76db53d25e04dfd933a" }, { "model": "meta/muse-wonder", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100754-exLwjepbLFAKIFfBwU38", "proof_sha256": "6ac2d25a49ae9dafe22a7cf2f9dd9bb4b85b4c3bf98345e78971979d132c2acd" }, { "model": "meta/muse-wonder", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100763-VZdpEVJaROsZVBOcXxB2", "proof_sha256": "0b3f41a1aab261e3ad1c6e722cefe03ac0b3ed968a70f8294a2ff55d306cf231" }, { "model": "meta/muse-wonder", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100783-BIKD1Il1kl1ChYqhaBEE", "proof_sha256": "bf26bf9ae6a436e7357475d3d521b555f1aabcafb2f7781a93dc7dac59d08eb1" }, { "model": "meta/muse-wonder", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100803-KIUTg7gtwBp1MQRmRCs1", "proof_sha256": "fd1c61d2b1be1d99796172623e3f5a2e5d78231e2ae713046436bf5619077abd" }, { "model": "meta/muse-wonder", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100814-57YOQbScSyQqXbCyGsE1", "proof_sha256": "5b2609797482e6596273dbca8a146f16b69f8400926907ffb93e6559324c9e03" }, { "model": "meta/muse-wonder", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100823-KpIBPLiSWv27hk0OPT3T", "proof_sha256": "7a100866a0e335cd23409aa09b26782a77f6c2ed9d984dbc65feaa4333535df2" }, { "model": "meta/muse-wonder", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100842-GSrKYXeXLCFCPp1qqN1I", "proof_sha256": "78e638c0be84ccfa0249f2de8d84c87015e7859aba9600db2f95f96495d5c578" }, { "model": "meta/muse-wonder", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100857-8esOvD6bGG5JEdPkMI3D", "proof_sha256": "35645a49bf21434e7243ef57ba3f131f5b41508d82055afb66c940659d75e9e7" }, { "model": "meta/muse-wonder", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100865-lOlhqPvFIReEAmzV6GgH", "proof_sha256": "594890fbac2c4ff20ba421c7b8cc4e41cc7f2312e1f12725ee77930b43a6822b" }, { "model": "meta/muse-wonder", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100875-sICPP7GQKNMibtXn23qA", "proof_sha256": "597410da8fe289d487f7ff4d44ea9c48075409b6419f39ccba55a78cf6b3afe7" }, { "model": "meta/muse-wonder", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100885-TytqDBP8X3yFf5SZizG6", "proof_sha256": "18da0db7cbfe1347790f4de2beb48fef0c05c57e17be188451ab9150968c3de3" }, { "model": "meta/muse-wonder", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100899-jnlfJ5AYuj01sLBh2ImM", "proof_sha256": "34ec8e5a7e092fc19c946ad54c18452d3a98cd38ed61bfd1d4dc243f855914f1" }, { "model": "meta/muse-wonder", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100909-pyuGkSdmt3DgVjpUQegC", "proof_sha256": "93eae8b2a821d8d0332423612e48602fcf4eaaad3e3de2ebd79bcd1934e0fa98" }, { "model": "meta/muse-wonder", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100931-G37msJirBpV8d7he0K6C", "proof_sha256": "dcd7ad59ca86fbb5d9a6139a36df3183d0f70d2083c4db4bb9d1bcb0d1bc627f" }, { "model": "meta/muse-wonder", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100946-H9V7isLhh8Lqwbm3fwdG", "proof_sha256": "8d37f39eaab158eac68033998f13bfc94b0e3b62a4770c1d2fe08171788c5fef" }, { "model": "meta/muse-wonder", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100956-6tdbWsu9NxmpoqtAX5ig", "proof_sha256": "40e7b04d46df52abbabfc2529c5be046fb56d2d593118c472760a60ddd94b455" }, { "model": "meta/muse-wonder", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100966-I5YclK2NxSEZoDhJjx1X", "proof_sha256": "ae0a0b81305dece4e4083d7419f8f319459cb3ee9ca3554008a8b5034fb0756c" }, { "model": "meta/muse-wonder", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100985-5jnm27X0Zb7foIKBPjGH", "proof_sha256": "4f912e0ba8566c8271934db7f75f8a7dbe3a96591c39020df08e1d4bca0fe54c" }, { "model": "meta/muse-wonder", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Meta", "run_id": "20261027T103825Z", "generation_id": "gen-1793100995-4ZFI0i5yOgLaz1nwZnu3", "proof_sha256": "5a8a862c689ad3369179fef31b52fac006317ca6efdc810f47e0751778ef1102" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097505-UIKTlxwjeG19FphikD0l", "proof_sha256": "b6501afd43aa61a65ed2c23a731c0679a556ad96845d622e812524b7e26cc1a8" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097536-a2KQ1Tccgh8mJWaS9gvn", "proof_sha256": "f62c4ce3a96a1ed7f8d6f7bd01f5ce6fb7f91d5997f42bf6bb8eea7bba84ae6e" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097874-d4sgGbh45rs28XwQtuMK", "proof_sha256": "36e4ab989cc42a231c503e1a130dffac054ce37224856a89bf5420603374b7f6" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097922-KahjBvyFInnWbiaOkPa1", "proof_sha256": "1643381549560be876a639266b2752729e908ec90261129c2a3e42d7d5b2447d" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793098049-GkR7Uyerew4c08RHgnZK", "proof_sha256": "af48723ec55c9ee9288636e885efbe385b9cfd4525f5ba6c20d598471b112e19" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099376-8kvinjZ0qqffZHmy263B", "proof_sha256": "cca5f394b64c59a60fce03e54d7c2f5fa0836968bd3622dfb8b33035264f2d1f" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099453-JsUtAOX97chA53IDhAsX", "proof_sha256": "56cc19c2a13c8943793fe65c401885a4a543d30bbcd1aa06cf72b55a7666d7b4" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099505-HAPPe9BMILnaoAvsg9DC", "proof_sha256": "63b6b59bb7576838f82f10e28478c640ffd119ed443221579b899aa82ffbe729" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099566-UwXTIo0nBoz4nYi0GkqL", "proof_sha256": "96bfe0e5c237aeb80a4329411f8ee867075098cdbb6dd0dc6d5ff4eb6ab9894f" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099653-FC1970TXw7MIZ0dB9PPg", "proof_sha256": "a196eb3d2f42d7545e1a5ec076c72b15c9a1291a696d317d274ddcf03f64bc46" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099775-1zU5ko1Jq8BhY3BrN6TU", "proof_sha256": "89595b45f6ca87abcc859dd9095980a15333bdfa955a03ac2bba65350b149679" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099834-CS7OrCmkkgVn7t3bksod", "proof_sha256": "1f57641b703f2b9cf981560f4d818a3eb6c476ca351278186aa6afa3336a2c72" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100242-z72ARymJn7ORPKYn6Ord", "proof_sha256": "8179a6be3a2692eabb869e17f92f26e3cd85f1dd0b3c119694a859ede50a3196" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100261-32aBxwzWdOAVbVXdXtqO", "proof_sha256": "f7ae1ae0fe0b08e64374d3431ee4bcdc92ebb2c9beec0fb2fd01a349830789d5" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100318-RqCmjxMf89WnrPTEm3Ix", "proof_sha256": "0b80e1004993b27f57f006ee4e845ff0808663c8c6f0c3f9158314a58cf0700e" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100408-ueom727Hbzy6eH7g3gRW", "proof_sha256": "e2c8cb93ba5b322096e781e929b5557e8be500059b433eeaf5ca51ffcf68c27c" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100449-XyEt09xABmJOtz3iUtnm", "proof_sha256": "1856af77829b28c9fac7e209b63762dd6704f5fe1396b6db7b16852f0bba82cd" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100482-6EhCnk1cXIGReZsO30hu", "proof_sha256": "feeda12747712e7eccc2ba5748fbeedd305fddcf5044a8ee3224a104b2064749" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100509-zEFLAfqOuWXotWnJeYNe", "proof_sha256": "d5d62f3e2387e8543825f31a67d0eaf10934d73762f75e7353b2899ac802fdb6" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100547-bSL6tCYhFjJAZW28Jb3s", "proof_sha256": "67026ee29a9a168e6f3348dd99b8180e56fe7dab7e6dc76f0959f5c647b71b0b" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100572-0m6SK9G1v0dfdZosdKud", "proof_sha256": "c75bc7560895fde173c77fc70e470ae2d511948a61f2e4e70de3cffe53431411" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100587-QbnTF2IhE98Y6FUxhxmc", "proof_sha256": "28a94dbca420f89282b0913c009767107133d11feefa853699474cbd4e5e534b" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100680-DycZQwui0kh6cLc0ayIC", "proof_sha256": "40af681441761f162887858e23ac1a1dbe3ee8fcc54d149a0c7d45e94485c6a0" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100815-RSEhvF5shyybwQ8YdSyZ", "proof_sha256": "86a2f4e95f8508a55335e5acf7ccbdbb22269c1e9a94dbb38ae44786b36d515c" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100846-7Oa29n207XD4gwXnW3EQ", "proof_sha256": "9471804dbd326a25836e9264aa84500f4b090034fe409e8c830f05ef61490503" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793101040-aTfxVkJ8RsMsskxLRvlp", "proof_sha256": "2a684f062128c7037db6dc2bf42058f675fa203c4ae8dfbe3d7789b489c95549" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793101358-2gusaJLpVo2maVAyg7sv", "proof_sha256": "47b07baaade35c1603107deab73f25774f8ae5a604d286ee903f77a059673b90" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793101404-ScrzFFL2F6CUQOb1d6Oc", "proof_sha256": "13eb264b626c1f88accf744fcf4a955b547c1dacd64fc09f8f80dd0fc9357eaa" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793101532-YfbtxQlGBbF1CFTPkhW9", "proof_sha256": "431cc8b4d2fdb9fa1b3fe3910c619cca84b9a943b3d6fa2e625f4faca2dca906" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793101589-TH4dJdsIumHy7p3DNqTR", "proof_sha256": "351663ab1fe9ed9d3975d1349387d98d9f7e506c08c0be42ad11e5ce8b1a26ea" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793102041-LrFV0FswUZdGJd27KzdK", "proof_sha256": "0c2d6ca5142b0814fe562cc6e9a182af00eb62a51ca2279b547564d606c96e89" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793102317-Jt5ZBfK6xHzkHXeV0BLc", "proof_sha256": "df882a3f6b4c5d753c850baf66bcaef3726d76dab607eed3746591bcc8abdcbd" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793102402-nP0t2bFkCcI5hcQ80871", "proof_sha256": "bd90ae41617599a042cfd4e800ada6e2a357502753def0f4737b1a84b968cbc0" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793102443-oCqPyMREAaQwPGFMNoVw", "proof_sha256": "ecf63edadfd6a5ea34bc4c880951c99f51573daacc1b47f26b1910c6281797fb" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793102484-EivSviij9ygECNWubElb", "proof_sha256": "b9a122690984bc3125b649cb00e328c2d7065b1534b0fbde4dda0a9a47ce67d7" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793102534-W8jwVesCheSDg4yFbLBi", "proof_sha256": "4e7c3c8276ebe016d6324a826049be649514b4daad127f5ce3bffa71e1cd7733" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793102595-3z4lvGcRKJIu1zlYZNpu", "proof_sha256": "b6eff80bd5e105842f0a75f263de49f442f266daf9f354613e39360da3f56629" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793102638-pVYk0wSgXK9I2FY3D13E", "proof_sha256": "13cba499133847c973460cf8122925a0b8ddc474905845bde1f95ce9915c5e42" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793102917-VyqbIkhenSvW5D7TSJ0l", "proof_sha256": "faae0f2c761f9a79d7c325bd0f0b0de95e05ce30543e0a8815e4b5dbcae6b86a" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793103019-dlQj6FqD0eri32D4agSl", "proof_sha256": "51a6b507d8c653d9fc135825f326696e167ddbf124681bd06ecbcd8546557c71" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793103092-9HzvJ35jLrUDjnAEOJiW", "proof_sha256": "95e26bb3f29ab4a00e3240989c0f121cb734654272349da41a541bf7fd081782" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793103232-FinpGsOet1O6Xt4qVHOb", "proof_sha256": "35f172f143b4118ed28f18ff7eb541f24356621806898825083679e13d9a551b" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793103304-40bcoXKtoZY0GqB3W6Th", "proof_sha256": "08b653b06aaa3efbca09aefb51cea5836de8ca3bd17af4f2eba6ef97ad9f9aed" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793103426-KkoooqLcozueOdWOguCh", "proof_sha256": "6a3acbeaabf68dda4805becf243678f89700f6186eb8edd4bf7a419ad69c4992" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793103679-cRHHIr0vjgzC519AnVnd", "proof_sha256": "c9d337221138ed1adb91dd65d669cb31b0583f4b17f1774c0283156ecd163990" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793103726-XBDcdpZj1P1O841IuRmU", "proof_sha256": "8a17c4ad0354ba9eed63f8115d8b6e7efd79119e45dac23335ce47c75efbeee2" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793103755-9fiTFOpHPg9sv1BoGzto", "proof_sha256": "5534c4e16812661ba0bc20e10883d3a2db53d0a691b51a7207687dff21cbc20a" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793103794-dwLHgS9FlxI1uqvJHL2B", "proof_sha256": "ce4bdcb01090f46a39a439f4177f7cd6fc942c44cca857ff7a81cdc9a4c6c680" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793122224-tVN4jFgOxkKJEnW5ji0w", "proof_sha256": "1f54783227b7e5ee37e7a9ec345a5f45dd9e0d10e37f00735b262b1d47909f17" }, { "model": "x-ai/grok-4.7", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793111034-a9SmesD2Yy1stCrtV71Y", "proof_sha256": "022cd1036cecac202a782715b4297f3a12b0f31935c6ef488bac40236c5c40e1" }, { "model": "x-ai/grok-4.7", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793113843-DFQHoZBbh5oRmETbbusn", "proof_sha256": "3ddccb8420eef78aa1f3dba2c3db9a6ab1acfe29a92bb2cbfaae88eadeecb664" }, { "model": "x-ai/grok-4.7", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793113883-vmrIB0UNUMFbKmSJ1F9m", "proof_sha256": "668f2f4d31cd12d2caea18a049f16f1fc3669e858e1017b711588e408633784b" }, { "model": "x-ai/grok-4.7", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793113937-Rchx7o9LiFBOD0TBU0aL", "proof_sha256": "f4a7f225afc72cc45437a1bed103afd4d87fce1bc342574b9c68a8e28ae451c5" }, { "model": "x-ai/grok-4.7", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793113988-LiXSE9tgkYceVU7hsb3m", "proof_sha256": "4d9267db5fa17ef9cec2bae55944962c37e5b891b9671de8cfd8efd0bf6a2332" }, { "model": "x-ai/grok-4.7", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793114324-tchTl5WGA16CkKpK9tB4", "proof_sha256": "cf92591b8d1728d802f035622f627d5fcacf19ac08e6b470c9824cc85f92531d" }, { "model": "x-ai/grok-4.7", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793114362-EGWULV1MA2HGssEbYgsi", "proof_sha256": "3551f439b9bd8d280e61b0eb4d55666db5c3b87e051ecc972e66b29284850026" }, { "model": "x-ai/grok-4.7", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793114394-C5281Q3JYVQQeEnjO2X5", "proof_sha256": "54f10419034f778b29f911da6f3c0a8f90a0dc42f1b2bb137470bde76642e2e6" }, { "model": "x-ai/grok-4.7", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793114422-Y8Kkfdmtube3VTq0QFEJ", "proof_sha256": "c8609aa9b96bed37720cca9475558e34da6ff80d3cc9a298aba824eba888f100" }, { "model": "x-ai/grok-4.7", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793114445-ocUwAwMtR5f1LwGOhESL", "proof_sha256": "e28b17cdcdcdc933570e01c32c321f27d1a847fc5738c8b89e17119b7133c221" }, { "model": "x-ai/grok-4.7", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793114465-rn7sZlBsVNJQsUqGmUFw", "proof_sha256": "2ad4b9b960edbc79ff177740f977a4cb561d4055b0e822aad77d0d08cbe0f019" }, { "model": "x-ai/grok-4.7", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793114479-26Wecn0egJ9QzXGlyEvU", "proof_sha256": "b574fb9eb6177297675dda1e3cc7b08427b6fc419d616795354a2e9ac26002fb" }, { "model": "x-ai/grok-4.7", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793114501-KhnYjaJ2ouxBk2t0OvyQ", "proof_sha256": "ef26a8bb9cb6f11c7918b7282544a14d95298e73796c8d5f49cf4179f6f138ee" }, { "model": "x-ai/grok-4.7", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793114522-mxnbLFJ85ufJbx80XiHb", "proof_sha256": "e7a64ca3fdde00504d24fd013bb9f96be28bf13125657409149401edb79da2b2" }, { "model": "x-ai/grok-4.7", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793114547-eu9nt6iMXeWpdSJlh43l", "proof_sha256": "6249cdecc7cad9bba714d81523fc1c0eab5a0cfba1cb4f1216b52d40d87e99ff" }, { "model": "x-ai/grok-4.7", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793114569-1GOozsevnIUZeQlsHEme", "proof_sha256": "4a8fa40228da56206598a26a2ae603b141553313ee07669303a4727876482bce" }, { "model": "x-ai/grok-4.7", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793114598-XjavFiq27uL6x9si0YaD", "proof_sha256": "12acbb35fe7e79c200db0d4e1c020793b5514cd65fb6e75f8d4688c36b3e490f" }, { "model": "x-ai/grok-4.7", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793114621-Eaev1t9Ox5o2ypxc87QN", "proof_sha256": "e241d177aa98a440cb337af1bf041f274fc03e601a934def123cf7cfba148944" }, { "model": "x-ai/grok-4.7", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793114644-VpcfXLW8t045dhqUdLWU", "proof_sha256": "b716aa8fc34682c8d82381de0b33ac0ecb905ec9e36baf8391a2d4c33edfa9b0" }, { "model": "x-ai/grok-4.7", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793114773-yPoZH4cMMF1BMLo8iqIP", "proof_sha256": "0d0a1224176bd84a4511c7a2e440066033176b7a986f025ded91ae42d318739e" }, { "model": "x-ai/grok-4.7", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793114796-uzPAnmD5gLMrSouTtpAJ", "proof_sha256": "b381dd1c912a72bb56ae54d2fc8a3d1999a8f4b78081c52b21463aed478ca98c" }, { "model": "x-ai/grok-4.7", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793114812-ChlVIMWjSPfkpSqBaAvx", "proof_sha256": "6ca64a5f28c02f3d1fe30d6f2d8fda563359bae696977664ce916389a6d29022" }, { "model": "x-ai/grok-4.7", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793114824-O2c99eOhPH6VU9iZqI7Y", "proof_sha256": "7f796e969e4b6f138aa6423b043b741be550d188dc0f7bcc161c9ef273ac8860" }, { "model": "x-ai/grok-4.7", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793114877-8HiopLJl4ArBCnDq7uqY", "proof_sha256": "a0ca986a2da8915fe52dd12bc83caba9c3214c678bce082f5a432207d7b3b146" }, { "model": "x-ai/grok-4.7", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793114887-bPIZgXPYrzqU1EOiL824", "proof_sha256": "bb577e1294abb615d5f7a7094315ed26a9382dde24098358526384eb5cc1f4c4" }, { "model": "x-ai/grok-4.7", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793114944-DUEa2j9mgdLHNMfIYmAW", "proof_sha256": "3cd7ca61cc8808b5458a7d5b1fb689afbb17143cd76309272318990946a862e9" }, { "model": "x-ai/grok-4.7", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793114962-Yy2a1yPFV2tSyTuLmpdZ", "proof_sha256": "241a58c491517a2fb50439ec8eaeac3d5abfa288330ff1a636e23980f20e4e45" }, { "model": "x-ai/grok-4.7", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793114984-ZSZnXKiCNJNh7q8hLYVk", "proof_sha256": "3f7404a1209088cf61638befb0d9977ee8f7ce88888aea77be54b20a371ec268" }, { "model": "x-ai/grok-4.7", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793115027-yxSm60jGWOda0n3KhIcX", "proof_sha256": "7e6b4c5fa86adfe92873e33ef3a5f9ce6296bb235c50019a4e9b3878e989af2f" }, { "model": "x-ai/grok-4.7", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793115038-34ZkYjNNELY8kWbG1aHY", "proof_sha256": "67fb0b62adc32de7c2ef6dcef87c03fe5c5273ca6f9696b0158446afe50743cd" }, { "model": "x-ai/grok-4.7", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "xAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793115074-mghGNNeRDaXoFvsHC1HJ", "proof_sha256": "a738b61b18dd1e6e6502d4bdc5197f461b810960dea7f98fe3003532cc30708c" }, { "model": "openai/o3", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097505-uktSbmiIwCCRlSzl11Uv", "proof_sha256": "6fedf8ee9077c986bcb65417b94bee1dd468dfd8431368c6c5d8a711edc80418" }, { "model": "openai/o3", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097521-5q5VorOnRCdt4tZXXn1W", "proof_sha256": "9a95c12f4a09263e294d1a3bb2c9e34bcde6ca820d09a6ba0925a4b37c32bc87" }, { "model": "openai/o3", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097538-Ta3vRoW6YiwhOck6mFfu", "proof_sha256": "a588d1f2f3505b3f8c92a485acd7c8d16037cc8119c7271ad7f76725ce6b9c0e" }, { "model": "openai/o3", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097564-dHeBIwxJdepfIO8slmKZ", "proof_sha256": "70d22799b57a8df101a37ad672a59a138dc986d5e0243cda3e10d0c04f11f9e4" }, { "model": "openai/o3", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097629-rt8cHYqvl3C0nMCIMtuI", "proof_sha256": "217cc1d93e2e54f6f1fd1ae01a42947e74a825222d588846ddea4ad522e2fce2" }, { "model": "openai/o3", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097663-8oOLNcuxRBfitaqJks6f", "proof_sha256": "08067eebe7fe3f362dac261afe46784ef95ab4642c83393d5d4a68eb82fa0f96" }, { "model": "openai/o3", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097690-Ba946vo1Le3zDOojINEY", "proof_sha256": "40588a31fb46d337330cc21757cfdf6ccc8846738c004a4ed8395612c274bdfa" }, { "model": "openai/o3", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097716-dQqA7ntOoDbb9XuXT3lx", "proof_sha256": "7892074c5decfd3cf9a4af71763793745a2aed8811c1b1fe3d11daa79cfe3917" }, { "model": "openai/o3", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097730-rDzx1jgpnLKaqg6IPOih", "proof_sha256": "7b3dbd8f837d85525e010c8fda2df9b4035fe6bf5b08ca2567a15716ec14cae2" }, { "model": "openai/o3", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097756-AqOy2Cbj98IUYiZUrKbA", "proof_sha256": "c8abd9b2a0567d40fc83dabcd5598f668c182c2c1b6cf81d293d24ad3f693ba1" }, { "model": "openai/o3", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097790-ueinEEGOWWmxgiuJnGt0", "proof_sha256": "2e6a36ed573a0d50afe131044c867aed736f86837124434a08912cd144e906dc" }, { "model": "openai/o3", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097817-04Zv39nxhN77RUV5sP63", "proof_sha256": "1540dcb2108227f2bb207f3ca10591e497ae0c7d68932cbdc55030447792a488" }, { "model": "openai/o3", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097858-aaFcnXyFVhBcPG0t8kOT", "proof_sha256": "e90e66be96427f4c0a7d7d4af44f93d65f44d81e238eeb993dbac465b71ba7aa" }, { "model": "openai/o3", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097869-LYhdPYtHIk6vxJ9DRn8M", "proof_sha256": "b05bd308bfdbcf5fa3724fddc1d094ac9e545efaadb3aca1e2b07fee4492ca48" }, { "model": "openai/o3", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097917-RHOig3L759NIsruRQTzX", "proof_sha256": "6944b2cd207bfe8330dae24de94fbc1a9f2dde40d5b54001062e517fed3bd033" }, { "model": "openai/o3", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097967-YL0tvvuQFYVGs3xng2jB", "proof_sha256": "d1db659d4e1e216e54e1cb4989ba09daa822427fde5ff699c2be2ab5db7a18f9" }, { "model": "openai/o3", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793098015-rOuXts8Pt8wgHyFAysFY", "proof_sha256": "1c5b6233a25f9a1d96afe0fc8153553077a9d5cee32d6b2483a1e526424e05b6" }, { "model": "openai/o3", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793098063-lFxtqAgCtGkmkDqZXUlw", "proof_sha256": "d31936ac2b8448fee3350bee8068a6e80529938fdc59bd062dc91a40ab90237f" }, { "model": "openai/o3", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793098106-b7JaR8V4drK8NH1NrxmF", "proof_sha256": "8755d092a82d4405e1372166197c481012927ca27021e826b1914cfed1658dbf" }, { "model": "openai/o3", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793098145-lAcVQ7Y4i7YFD4Di1VRw", "proof_sha256": "a785423acc5dd35c42771bf2fa75ebf1137723e466c9b8e88cdc9df6eb57e6e3" }, { "model": "openai/o3", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099376-EcOOjDTrNqOltK8Z1M6N", "proof_sha256": "e23b1c48b7651478e477e878d27c9c56392d085a306e363e12235c6bb4677d82" }, { "model": "openai/o3", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099418-3Qz73zPiyTaScIqstdWX", "proof_sha256": "fba5caae046013dfc5f8b2782045207572cd48e4b25ff6d57235c80b79aa0b3d" }, { "model": "openai/o3", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099466-yTCv50lfW6O3K21hM3Y7", "proof_sha256": "b83a9f4057e51725f7acbcc800334d045af544290fd07a5eccd05cfe1c727266" }, { "model": "openai/o3", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099562-k2eDbSKPXETExYhWQyOw", "proof_sha256": "2605722baffbacf22e24a96b0de62513b22e64dea2a5ea5f7ac97751bff563de" }, { "model": "openai/o3", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099608-pVg4ch4FvgdJpzFZ3cL0", "proof_sha256": "745650023ec67dafeabf19b6712f23dcd32e3411d52388ce381243b32f55f185" }, { "model": "openai/o3", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099662-U0CM9zSnfpkCK6Kmmjb9", "proof_sha256": "4c5c3e1ec950d450cedc389f995067a87cef5aa3abfe9486eaf682667d6d2092" }, { "model": "openai/o3", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099714-EsElD7MC3K3028rtcAdP", "proof_sha256": "fac9cd392d018348213457720c57ac6f435efae60679a1588fba845d50651995" }, { "model": "openai/o3", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099761-y3L2XK0PUpTMCP7kDa55", "proof_sha256": "18ad5af4f8caf21f582280714d08e5c48a27f0e484e79ae9107ca60f313ad217" }, { "model": "openai/o3", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099812-P4G5JoLjDbuQcFACvTFc", "proof_sha256": "77d0c4a682a2dd25891f5897b5529c391af3799fad3ecba84c86fe85a244bf8b" }, { "model": "openai/o3", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099863-pDHZT5Nrv7ECEwP5kDcC", "proof_sha256": "7677242848da9aee9961c5238d0356c3984cb4830bebd5d23ebab25369a7a045" }, { "model": "openai/o3", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099908-KkJXnxLRZglHkSzD8lrI", "proof_sha256": "f318e495f532a884651e185b1bfcc286fdad894f0a27bc8d5b7c688a36bb0dd4" }, { "model": "openai/o3", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099954-owFKhItUf8uIswPHTTsy", "proof_sha256": "294459d75baadc8b5106143e31eac947719b8eaf3f3ebc88fa960e64cca38a69" }, { "model": "openai/o3", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100018-aEv2ONChPHIpVT4Tq4yb", "proof_sha256": "7653edf5bde2145642b49298231dda123f0de6ac7377e0f699860cd8536dbc3c" }, { "model": "openai/o3", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100052-rqLJt9j5iO6GL0nKr6Pf", "proof_sha256": "2c0347f191192bf50ab621663f5078fc444804f70b989ea3b873bfd352bdfee8" }, { "model": "openai/o3", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100105-fB6ZSQCJ8zn7EhMhTz1f", "proof_sha256": "1421cdb400ea52c413c23b0986330ae4c7eb5f12d5da0320e3258441081c4a1b" }, { "model": "openai/o3", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100156-VfPpxnyPXAbedyC8xc3F", "proof_sha256": "020cfbb38179882e0ca2b5e9b4a60399c59916db3348d64ea394f0932f481ada" }, { "model": "openai/o3", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100209-p7IXIjEvG6WZXB4fr0Qy", "proof_sha256": "804fd13dc22caa40e22a32818f53bd3af9d2de325052ff203d44df77a41e7452" }, { "model": "openai/o3", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100253-IGOjOyaMHTnAtsu6Kl5r", "proof_sha256": "ae5efad6103b66467384f3d3ca18ba14341487fe1292a94b6a75ce24cfd644e7" }, { "model": "openai/o3", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100298-XoGAnaTQZHp4Fu6ULgpX", "proof_sha256": "d0e0450b8dc9d94729ef729ba3dc041402ed3174423e7558098147fbdad907eb" }, { "model": "openai/o3", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100346-JTBK7L96GiUtdEWtq0Qu", "proof_sha256": "d4bb102a3f9c660a23ea63c4b88fe13a7f5e27f323ff4b3030de58b987ffafb6" }, { "model": "openai/o3", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100388-Lj48gEOWd5WsLOFDaPRy", "proof_sha256": "1b3c3994fcfa2fc8bfa2b7fe72ce65eb26c0a40efefa3384151df8ce6324d6c1" }, { "model": "openai/o3", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100413-u264eKxqHxnPpuaKDIJR", "proof_sha256": "0a53bb954afeadd7d828b308a6ef5962df8b21d52dfe9fa3dbc8b34a70d60f26" }, { "model": "openai/o3", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100432-k3leDnJtLbT3jZoTTsLM", "proof_sha256": "90d8aa03c55bf848ffdae4a44ec57b210b6ccf05bcc8344fd144fae45b302947" }, { "model": "openai/o3", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "≈ 4.9 × 10⁴ g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100454-VEDVZFM3vFURS14QYs4h", "proof_sha256": "396fc151143924a38ba590eaa610ab7ce7a4a27ed0ab2e52ff431c234ebd711b" }, { "model": "openai/o3", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100481-AsIMFCzy0QusqvUYbGSb", "proof_sha256": "b222c7fd64b739ced28ae598dac2fb5c8928b5abef9a9836d1a352eba4df4961" }, { "model": "openai/o3", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100494-CpzDLLXFZZZjYVYkrGgK", "proof_sha256": "a89bd0bd03448829ff54e54fbd99c8ae40d2f6ad2946810b8d2ed43ecf9d02e1" }, { "model": "openai/o3", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100515-1RGEZWBEQMPOFYk74EbA", "proof_sha256": "17c61577be7c4a5063a9eb07795d9199cca82646a7f49a11f5739c806ae5ae0b" }, { "model": "openai/o3", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100523-k6r50CQFnAHLRd077d6I", "proof_sha256": "4b0c2f18a16188dbda81862c788b75c261c4515c8c7fea9aa551e10cff037050" }, { "model": "openai/o3", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100575-szPtmJq1Wi2DcZ8uhwsd", "proof_sha256": "73f561176dfe3d96aa7f5c28f1579348df0c2414484fcbf03b4e5e5cec134db8" }, { "model": "openai/o3", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100597-XoWRhxGFonZJDHDNTb8k", "proof_sha256": "ecff9410ac8ef1f6bad021b5818b3b4c3affbd1d8d9d0f230611bf94e3a97d04" }, { "model": "openai/o3", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100606-V5giyJLWV5a7EJkCwUMa", "proof_sha256": "145d9decab26410bc60f3cb179130178d5824d46c3b1091657757a07d88a026e" }, { "model": "openai/o3", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100624-nnAQ7HH9psYHiJcRPIDU", "proof_sha256": "240803db241c1ccf2ee9e34010dfd4bfaa496a7d1f8fdf05e423321fe6465aaf" }, { "model": "openai/o3", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100634-UyAjUflwP3TWEK8FjLwH", "proof_sha256": "fa45a32e279fadfb7260d12c2ef8e5f4dc778079a00a8d5529270b8e3ea7c574" }, { "model": "openai/o3", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100656-wDG5o5nMqcGhwonwLOVI", "proof_sha256": "3ab205f22b0589dcd1822e85de48458554b3399130330237cccadd3df75e76ec" }, { "model": "openai/o3", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100678-DNtFmUdtLi3tgLJJpMiI", "proof_sha256": "c98d5607dea719c034f0f8a3c601258d8b711e9f64195d533e8e3c4566367857" }, { "model": "openai/o3", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100690-KCcUpXd2vzoeEbF1QOS3", "proof_sha256": "915feb70c6a6393a3e129b8038b5706e9893ed44ec2dd8f7dc1e6db3c737c62c" }, { "model": "openai/o3", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100702-4kEou5KEjoKFqJM3IPoj", "proof_sha256": "17e4e6b8b337089e90c496a18ba801735c4e89f546d3bc213fe29516063e27d9" }, { "model": "openai/o3", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100720-73Nidr51uKBMfBQ3OxBz", "proof_sha256": "c9e89d8d2783f4a8ce78a5657be09f9621ee07f7876dca0bf474fae6ff1f5d30" }, { "model": "openai/o3", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100730-4baw5mqKCz0zPRJC1eXp", "proof_sha256": "b337104d33b98e1d5b2a6a4aea483e72a4e09366562ddd168d2b59d1695f0cc8" }, { "model": "openai/o3", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100741-jCr2HltwUNysWh4DzDv7", "proof_sha256": "a36627c2dd267f483006a5a84f7675a9c022c975651c0a7fe0ed34e7ab6d9fe9" }, { "model": "openai/o3", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100749-rTQS1t8HCA3W2J3AoYds", "proof_sha256": "4a67a63f02bbe6f34299ca2583aba8af685f81e3374d8f93f67c4f4d8e239695" }, { "model": "openai/o3", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100759-BqCMQAZF3j4kCObKaTQ1", "proof_sha256": "c63e4df2ec0b07a440c2097fb16a9bddf8aeb85911753039da12adfc3423a76a" }, { "model": "openai/o3", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100769-a8VJmjxrCzGG2J6W1Qlg", "proof_sha256": "b7062ecd3b3d7f59791950a2ad3169a1508c0713b10849940f9ca42d094d03a9" }, { "model": "openai/o3", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100779-8drMN8cvkhHxNorUE7dv", "proof_sha256": "1d36eef69e57b090b9dffdca036eb39e36fdbafe8ae096bec90f7d1aed140746" }, { "model": "openai/o3", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100789-SrQTLxpiLHfbZI9XkLVa", "proof_sha256": "1c51860cbeebb14d10c798f957e5f0b0ff510097b8129379d0b163044841ac39" }, { "model": "openai/o3", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100798-jpLQN3IZdNWAvkKNbinT", "proof_sha256": "67bd58d1fff9f77c84a73e0c0cadf3c741db4889036527d8d6a01ef0f41597b7" }, { "model": "openai/o3", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100808-jynlxwXgftRUhz8X9pe5", "proof_sha256": "6cd5e1945134a81edf94ceb41f57d0f5f0594c48c286c2637307d2f1abfa4d2e" }, { "model": "openai/o3", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100828-Dw1wXox0z5K1IcthNBH0", "proof_sha256": "885b49f7ffeb5497906467e8d22ed570ea6543acbd4001508c3700cb67c8611f" }, { "model": "openai/o3", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100849-gzlPMzWHuYHBMceqTc8E", "proof_sha256": "1c568d6743701ff6517330f315ec59ca0d1dbcd3390bc0589255ea2646bcd64b" }, { "model": "openai/o3", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100858-bUccuekZoJE5vFW2XqP7", "proof_sha256": "6f2c7dfa10b6ba2e846122a44b51875ad0e9a4aef2921b20f4f9d3ed8960927f" }, { "model": "openai/o3", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100867-3xPPVm8y7l48eRthENgS", "proof_sha256": "d1a29ff92b1e1696eedc04811a68e38ef7d83368d0fdcee13e039847b63ea073" }, { "model": "openai/o3", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100877-7CiFH4r9jC0YCrmz8Jta", "proof_sha256": "28acfe953429ce53f7b66b11d322d1708cd7d165dc389497941ac761056c2561" }, { "model": "openai/o3", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100897-i8GOg7rQD5OrzYbpfuTR", "proof_sha256": "d0382806a491dec8303453196f45401543f0527b40aab830ddc19adb153c5b33" }, { "model": "openai/o3", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100916-pIDeKhJIIVHxbKG0E1Mp", "proof_sha256": "d8b1315fdef5a7d25d8366e20e81c37678ccdc45709a26b6c2e8b690ee765ec5" }, { "model": "openai/o3", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100942-iHAL3ChCAzEvyhUDe4m9", "proof_sha256": "c6a12f6c90d5b6d54d98e68f0fa2167a73dd6296b0c79dee529ad9f62d493106" }, { "model": "openai/o3", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100962-brjAUvJR6Kw1lpcqD6Oc", "proof_sha256": "40e6dc1a880c69ce3989f3b9e8eda72e7f5ac9838907d44cfd206a74ff5b5ecd" }, { "model": "openai/o3", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100975-A7TWZo4YaYtXYzYMBnzr", "proof_sha256": "c5a36fd9b5e69564addec695a10cb3756fab7555762e782576b4f36951fc37ef" }, { "model": "openai/o3", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100995-2lor2RXjYKZVEoEL2Ccq", "proof_sha256": "911494627d3aa1dd626f8481b1be199733fff58c848085bd4525d7ef7cfa8d24" }, { "model": "openai/o3", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793101004-0ZLhQt8oHFX0vR2ututl", "proof_sha256": "c44f4a76de83d898f308bfbc4d1769baf0f5f2f28147bd7cd899ad7c8d63180e" }, { "model": "openai/o3", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793101013-I75u8qqvmmJC6TJPMHU1", "proof_sha256": "fbaab202c9a9a00480508b8227814f7906bff10673ba66280272cd2b7416f4f7" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097505-yiJwmKlrerrVXauPdyWT", "proof_sha256": "d7579a5f657f789af92d1d4672fc652c1f287ebd5277de1824a046c4dd3da756" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097517-o2XUA6ESCJMWMIUovIkE", "proof_sha256": "abfafb2331ef3f9537f4308561500f60222b7dfc32a289a397817b074cae2801" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097534-l1PFVwhnOn3MnTAs4GMe", "proof_sha256": "fd6e846061c43d9f57eac2bdfca14ee19bb56d55fcaca73e0e52a73048530ca7" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097546-j0uAtrfPNY7D0ADA7hTv", "proof_sha256": "33becb0b706284e2b8e02928d5fe60ce99cfcc68569cd8e2dbfab2121ac87b3d" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097564-RpLxVSolXnXq8aaVviw3", "proof_sha256": "1d8ce9c0ab571b33c0b98d1991c1fd6795654013289d1fb9a463e1f42a3e7e14" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097597-HIb9MOmGUDIeZUuac4an", "proof_sha256": "33b92fe57ab5ff1dd330082b4dcee080dce245b2ee19ed907d6bf3a5585f6aca" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097620-M4XUbhqgs48dO1g8A1qd", "proof_sha256": "4f534b0c176cb6b7b42604e8d1b43acda77cc5fd57317912bf25f4c3467cad2c" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097633-7wfCyS07HwESoWcVXyZT", "proof_sha256": "9b517108fca9a94c35d28a52b4b0885b753b54bbb3da6947d5f3163ef207c424" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097653-CbTvLC38f2oQyHhmY0sG", "proof_sha256": "20b6989238e7201f3e91a12e1eae6f9fda3dc573d454b6260adc3d2bf4c3f3fc" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097672-XsrpBP09Kx3k1PQbqumL", "proof_sha256": "45223277c6d0581af44fb9fee924c63bd54b6eb7a5becbe2cc4264f27e268ed2" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097687-z7fCnMOLSbF7VlkUdHxS", "proof_sha256": "d21b3aedf9eb823f7d0186272e27d982dde6c55581db38a42d7b70b7079ce530" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097697-pYGhzVytwYEQetY4mMex", "proof_sha256": "0c9549852dcfad3a504858662bb6ea96e885afa497ed42c5d55062ae80c84fe6" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097732-HBNNMkx0YqptmaWKM4VM", "proof_sha256": "36c6febaa4da4fa222a597171eaa7ac1adc31d83a3e6c5ed7d9ee4991681ef67" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097742-ooNRigurYTqspo6xjgrW", "proof_sha256": "04c2707b01d2d0779b9e5433c9a73ac1bfa81520734d156358600ba268eb3aed" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097763-aMEl3t0HAcXBSjYicQIv", "proof_sha256": "1aeecc82595704f6ac4a2277cab6aea727b9c124a6258bef4a857681673d886c" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097786-sII2QIl9fAD1lpwWSI4o", "proof_sha256": "e7c73404fa811c3bb83bcaa96d8fd1ccfa667e5539b649f4aee3b70a308ff55b" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097807-EHD4agrKf2ZTzEROXMez", "proof_sha256": "dadebb7a8a76335863b249c4a083ab8a8b6aee1045de9ed3a592520aa437746e" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097826-xDPtXzLZo8QKhjUfC1IV", "proof_sha256": "4531acde6ec7f7c540c7fd3dbe30ca011d7edc607ab0b7b22b9a83c33714eeaa" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097855-h0etMYEb1yAV5yWLojSZ", "proof_sha256": "cccee2d64ee5ad8c9e14e0e16e9566555cdb78f32d2ad3fd9a199cde38a98eec" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097878-aXjsxKBnqve25WwhG9cj", "proof_sha256": "2a4066f668842b965f2fd2702426679ded82c0b8cb61a645c8d362643764ccd5" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097920-PGuOrdLtZxaSp8jMcDOn", "proof_sha256": "432537a0aea50e7cb3253a875fa4e0dd75fe5f11b2b6b3d222b42c1e4617de73" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097963-lcDkihyPBOPp5w1ry5fj", "proof_sha256": "4294c88c284f4d7a6348aacfe630dbc8740f8517161f4b63ec4f41c866532486" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097975-HYZCSBb2BjKNpE6e5PpK", "proof_sha256": "b9c08aa26934fda96aeac8c975611cdb9bec0036d5ae61c3274cfdd4fc2d77d9" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793098051-QTlpwAELGyxhKQsnbrYl", "proof_sha256": "6ca1820bab1092259f5e96b4b890ed0541671931533f64649d9f0cb8514a305a" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793098095-fRPnHnCQx3tZv5xnzct3", "proof_sha256": "54299ab2a3ac6bd2810e7f2b1155bc84a0078877e2026479a8c002cd3d2f1a6e" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793098140-zEof7yrw2HW04VugvTd7", "proof_sha256": "29dfc401a3068024e90096acc90555c1fef8e1b0782143761471549106400826" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099376-Dnx0HBUl2BrWz5xCU5sX", "proof_sha256": "a9d829679d9d4cc9c5be8b38cb08b54530a18b84601709284cdb5e9a05c0a96d" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099420-nXhjmDxEOU8RWDLDVrwb", "proof_sha256": "5e4c45f07a3e4e61be86481f4c83a49b1465981e7a8c60ee816279d01e7fd65b" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099495-Sr45LNGGMr4YjpQrPraD", "proof_sha256": "fc0a6d5001a76315c578e96a7ec4854e2f478c00751452938223d47b0ebe5aa0" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099558-ha8eb9mJhCaBwgMM7fUC", "proof_sha256": "2dc2bfe90a7660929a9ca5016e5fc6786b6da0d84accbf7400887c2199028548" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099601-dndjEkjFchbQwQk6TiGz", "proof_sha256": "a04874f4e4a380d85bde61908fc0ef5c6dbfe5026383e021c8458c6c10af6f0a" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099660-W5SMfDT0sD5lBymFlbPB", "proof_sha256": "b24eb6a388ce8afce9524092ce6fde882714c01e2cc892247ba1fd8652bedca3" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099706-CQ7BQF7rmJwj33N1pm7G", "proof_sha256": "cf3e97fc4c5febed35c02f5d361ee3ee4e8e405d389c32de007ccbd2a0efd388" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099750-X1dMC4pTjCeursCDsZ1y", "proof_sha256": "bea7c0373922152fc3671f88f67546588b91ba1fda613a242c212ae73849a199" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099795-EINyAQM8b91MgfJhjajR", "proof_sha256": "8ddfd04d1f0bfc697820b83743de01273b7d55b891a1d86b558ddb06b29b30fd" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099837-36kREFuqP5oOSTwrfXDf", "proof_sha256": "6f88e44abe6f72dc0a582f5377651ec3bc865a83fb63a24541d77979e00cbaf6" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099882-UXYIj6Xg1MeoO2uFYay9", "proof_sha256": "542d68e4573db769376dcd7475596312668c0229510130b97a68b5bdfb2e2746" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099907-eyjh63uOsoV6rAhm9k61", "proof_sha256": "8eacdebd4bf7b0076603e17b4ac096b7f843fdbc824637297544b1fca3a3b13b" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099948-GeuXcPSWWDNsfdoazF3f", "proof_sha256": "188bd165023186eadfb20366f3facfd9c33acb72a26f6b86f2e5993e02d71f59" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099991-gLeht99asvM6p4jqpT0D", "proof_sha256": "a22dce1cd8db965a3b2af581734d16ce6a48b00da2e57992a82d8368fbb4f473" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100031-CuKZzm45XDSYyZpptcWB", "proof_sha256": "1a30490191321ef9cb2042aa6281a0bb4d375b0da595ced8be7728dce8348a44" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100075-cHNSacoDctDBOMXzRmng", "proof_sha256": "046440d8b67759a713c1c4e74c3fe9e0043e4185dbf1662554c5544b803af1a0" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100117-sRnMDy9Cxmfz8lIA5smo", "proof_sha256": "52c7fd52a4d6b30d6a4f9d150f1f3ec40b00e570e5784e0b7a2052f915c617fd" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100158-jlznJv0keAdLo8YKabMk", "proof_sha256": "87b43d34869210b38e390efc0c37f985aad494b3a4f97b09dc9f71cc02b4b1c3" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100225-zf1v69HtC8A3AJAlVUe1", "proof_sha256": "54ed410d3a5c2dd3dd86e7cede3e6ff583e4381c7850e599feadc7918db5c14c" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100258-NdN2CWe1POFMOErV8xN9", "proof_sha256": "511a6aad6abc7404705ac04442e0ac87a3c2aa94aaacb6e7fe0372ee9da9a931" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100301-m8zDwQw7Ao2mvVeDAbfa", "proof_sha256": "bc7834a2aaeca9156fb11251c018e42db9d35f90b50ec445cf7a0174df21369a" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100341-zREhRwZv9VnE2qWqHv4z", "proof_sha256": "a318396f374fb335bdef4e6a8383867e2088d84d787adbc65949d3821b37bbfb" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100394-38hcVVA7Uc6RZLCBTl3Y", "proof_sha256": "7b1bf3647e9b4a52166ce36e5356491b44310165a9d514103cd40dc0a7bf1124" }, { "model": "openai/gpt-5.6-terra", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100434-N5Vd78Ee9YaWdt3gddCx", "proof_sha256": "be0dd639d50fbe6e8932d415ec5eade208d3bacf1a27e1a26bbd0b04956e2c6a" }, { "model": "openai/gpt-5.6-terra", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100453-dFNiroRVaXcv9V8kaaKw", "proof_sha256": "86ccb015403834969588da77753c64bfc1efb9907707c1893531155a7e67ccd7" }, { "model": "openai/gpt-5.6-terra", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100477-KE2TxQ5kHWkVktWBsFcn", "proof_sha256": "e531a2681fc303f9bc97c3fc11d7cbf98a83e82cfeffd1208f80305d661f2801" }, { "model": "openai/gpt-5.6-terra", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100500-PzXYmhIyMTkU7Ppy0JwE", "proof_sha256": "a34e812903e74d304b20079ab2fb46ca7a5a216638964e52710455c120507a78" }, { "model": "openai/gpt-5.6-terra", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100511-HA67HJxojCxsfrDUAWBC", "proof_sha256": "d61912ab148c805592e3ebcbf45fc3726e0a2db03600aff5abc8a9d834b4991a" }, { "model": "openai/gpt-5.6-terra", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100538-1CgGAQH2KeJ1mBHNde7b", "proof_sha256": "8715ec22b4367e800e293ab18df91a459b5991dd3ab2c1fa8b4950801253b934" }, { "model": "openai/gpt-5.6-terra", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100559-FaD8rq005w50WKgCtNdE", "proof_sha256": "4520634dbae1beebf5aff5e2a3c235b77d09496a938e69d2c72678985249a421" }, { "model": "openai/gpt-5.6-terra", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100580-qYWyhAW6LtAZ5ZZwREWl", "proof_sha256": "543b18ae8968c2bfb05d965c15067cfe37882d849bcd6276e7f2a456f531515e" }, { "model": "openai/gpt-5.6-terra", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100590-V36zS2WdNN41LU8QcVTA", "proof_sha256": "e5fd143e2e4cdb71d916824581c42a9037cc2d19cd1640eecd5f1ebe0e0f598c" }, { "model": "openai/gpt-5.6-terra", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100600-2zLdWjvCwIrgQW8HGqrg", "proof_sha256": "cbd5a11fece4373a5be77915ad3fe66e73b642e28bcfbf16a12b1ae7188f28f2" }, { "model": "openai/gpt-5.6-terra", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100610-3YdngPsMhohguf9mZdpV", "proof_sha256": "aa68fb12637880da7cf25ef9fa1c45d3f1fff6072bbb9195278f2066e50c9d8b" }, { "model": "openai/gpt-5.6-terra", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100620-MhYx7vo8TOSwsBkmfuhd", "proof_sha256": "17e2975b6cf09960d9d2adf3edfa97539e6bb351371d0d47ab54b39faf6a6676" }, { "model": "openai/gpt-5.6-terra", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100630-8TJBN2uGkMdfUlMDenam", "proof_sha256": "78630128d715eed808c3da54faea94cae6e9b7f5473ed29f121074ce1c9d6456" }, { "model": "openai/gpt-5.6-terra", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100638-aB9D1Ak7Uvs4qNTYELlu", "proof_sha256": "219d739b3c32cedec43cdfe664163261d2463bdbc04b3231c83ded1fe7734706" }, { "model": "openai/gpt-5.6-terra", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100656-HcjUlHbciHbam6bBiHzs", "proof_sha256": "0e00b1ecfa2806962b68cdd68d4cf9ba3403551f89f7e37573757cdf48ca2b7b" }, { "model": "openai/gpt-5.6-terra", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100666-iWiSyX6hjtjgFDV9vVO1", "proof_sha256": "424411511ddf3f4fd6a089c7af5ef0a458500befe5c393ebe4d6a1fb87e6a48c" }, { "model": "openai/gpt-5.6-terra", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100687-PaS9Reowc6NNzjg621Wn", "proof_sha256": "7c556c14227e072feef2c70f67b158986f2bdb7a187df956ba25ebdbabaf93f5" }, { "model": "openai/gpt-5.6-terra", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100697-EnL9vM8SgkOje8ITGrWJ", "proof_sha256": "fc1462bb6c0fe3b4fccd30aadaf962b48ba63475b0f966fb8724dc58a1c82b05" }, { "model": "openai/gpt-5.6-terra", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100706-BsehcRQrHrEtZbpkCBc9", "proof_sha256": "49e24f7e0044646ba6fa87751a3f072cdcb5951b6b0ecedaf7d2417919960a51" }, { "model": "openai/gpt-5.6-terra", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100717-F8dMFtQN9I71Fu8neCg4", "proof_sha256": "183ae0de452ff55c10b792a818f4d2f57ded96fb9566b7c938d139d6bc87e006" }, { "model": "openai/gpt-5.6-terra", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100737-3ORADRmE77vyByRctJGw", "proof_sha256": "efdf2070770697484609878107045f401ed66aafd876318af6782190379f8a26" }, { "model": "openai/gpt-5.6-terra", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100745-5QaVoR0oj79WMZUYuJIz", "proof_sha256": "6d2dd072222ca3b0df4d8d6edcb36234dc138126ad16f28928e032b4575cf555" }, { "model": "openai/gpt-5.6-terra", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100754-GHtklAhL0TwVwR0rGQm6", "proof_sha256": "ee95eaa9afd3a53a03da14ebb097a55a2e5e0381b5e9614ac1b0400a9dd6a3f7" }, { "model": "openai/gpt-5.6-terra", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100771-st60PkSfmKg9KFmljCiL", "proof_sha256": "866da6076fe4f6a1bbd25817aa8037e7212b7f81d6b3600df15cc8594d788abd" }, { "model": "openai/gpt-5.6-terra", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100781-7LVEc6nbZEBtmCGqRXxu", "proof_sha256": "e45329165caeee2c9e79af10a8acc94fc137418cf8d21d129505a451de466272" }, { "model": "openai/gpt-5.6-terra", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100804-5DB2QYBVnSf6ZNetx7oO", "proof_sha256": "73e39c152845a90bc536ec9ae210247192766e25589f35cc3ef375a65591cc34" }, { "model": "openai/gpt-5.6-terra", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100814-8ymV9vnGVX8ajxINj6NM", "proof_sha256": "1514b63ce44f22d8600c287e29bf1b054318cb7b106933805ce9adbc02b51372" }, { "model": "openai/gpt-5.6-terra", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100825-Ch4d0SOYVPd6QYyioPgH", "proof_sha256": "3e2939e38a05f87240f5606fb4983e36d85ab0fcb2bdcc3a9e9302bd559e059b" }, { "model": "openai/gpt-5.6-terra", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100845-Bzt7d6hosdS1zTNEAPPL", "proof_sha256": "5565417634abe041938a478bfc54f93096a3b2936cce847214df4cd074e2bf03" }, { "model": "openai/gpt-5.6-terra", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100856-MYkmwaOp89gWEv9CfDPp", "proof_sha256": "6f77f4bae84570eb157e3c7e08f173757a9353c633f21ee696ba14cc22638cae" }, { "model": "openai/gpt-5.6-terra", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100873-EFCGsu5Xhc5gxKwaTVA8", "proof_sha256": "d8a85f06b52c3f13c392cf087251689cea37b32e093ff0ba2881c693b08cf7fb" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097505-MlL0A3mKWakXhuXdfZAh", "proof_sha256": "115d4309c79b8b7d9a79ce8839f2237a9b89d75fa001de4adccde559cd2fbaf7" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097534-4M4TdVxr6ACntZUBVblk", "proof_sha256": "c4fce15dcea944d4952f758125598e431c2465404b27fc598c71eb9d478a40dc" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097561-d2inG4wURPlTPUHx9fr5", "proof_sha256": "00de9930a84eb37ebc15cb3cb45c90b0937bfa559c4cebcc6d55fb363e9f13d9" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097573-ya3AWFH8MefcxPRKQLC4", "proof_sha256": "bb7d71201a38b671ab68f497a9718e1c731b75f7cc1b849e459eda8ef9b91e00" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097614-TurTIGDS6AP7bgdFvQbP", "proof_sha256": "cf2f8759cca4a58c36e6665bb243122667e8da46d46e43049d0de8015f0fa2b4" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097684-ihpY03oAIMFw4zkwIL4s", "proof_sha256": "013806d8320a0a5c8e8b30edbe92fbd7a06ab6b5426955a8a4bd9444e3271686" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097714-2439iHfyLn2vMyC9ib6d", "proof_sha256": "cfdbfdf5066a403162f893d0147b40253eaad9259574f7c7a6061e242396f3f5" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097739-zj62MoWqSUN4GlJnIoUS", "proof_sha256": "7f556e4237f103a344125d1ee25ea31c2c5033ab774cc6c655d33f08399503ee" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097764-FmrMXSqtdTe7AKiGhJGr", "proof_sha256": "b7c76e5a909890e66f6a594b1860f8dfe2f31669f1ec49872e7930bb1f15bff6" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097789-zp0DQjyDFQ8GmB4wiP1o", "proof_sha256": "f141988cef72adbe7d8e8710ef5285ada44e43e438dc75e4e915b8f29a7f26c6" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097845-71GLuyVrM8deajAXay9c", "proof_sha256": "7df8c38c98bd1c504270e2293adc507e8887460ec1530466a657c862742ba35c" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097866-CaymxSuj0Sxx3R3DRAY4", "proof_sha256": "91432c5d09dfb0f4e040f76e1c0354c1ed510dd9b298a3c6f8049cd7a382e79a" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097955-Rh6BbKOrutbmajcVipsu", "proof_sha256": "07fecd6ca4613f95c2cf330043a12003ec84248f85736f237378d4a7323d118e" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793097977-Nl9H3UP5xnQkJAXRWrds", "proof_sha256": "3767c4bf455b9a1481dabf3241f1de42eafb4170e93080b496865d476f099556" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793098028-QD2mpwgUFxoPK0EspFqa", "proof_sha256": "bac73c3970bebc48362bd25c06069caf4dcb9662a335f707dca4599d5ba1f536" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793098086-xs7asZFsKaihDJx4xxeF", "proof_sha256": "8047638793c55395aab85c514639c55201d04dec7d3c61884bd513b0df05d631" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793098136-GkAvP5Ifze0QquQ1jVRt", "proof_sha256": "93e4c90a2d14fa3d77e0baae7b1a26d9cad981e0dec1a04b1aa5f93f1513a690" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099376-Ogtn64cHY4a3zwkl3fd0", "proof_sha256": "92b9952a9aeefc6aaea5339d101516bcc9d33701f66949b6e64fd999f1cd3a48" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099427-S04C0ZTNviak854i2bKV", "proof_sha256": "fc2d7d6881d50c73b9b554284bb22db701b406841f9a8407edca6b00cbddce34" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099476-U6ME3dQjx1TkWL5qdUnx", "proof_sha256": "eb5f51095b98db21e2d94279b5427c0ba6e2993f06554df142c16f68fdab9414" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099512-bNS3wkMEGmkyFr7HoWvX", "proof_sha256": "1fbf30ea8f77def36fc52f20909869d4b92b21f8056edaffe8c8055e3d48ed93" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099556-RL8AouOuaRQlN2v7IVI5", "proof_sha256": "3492b5c50ae2d9016d9401f51d57ae0ae842abc23ba7e17cedbc1fb29259c431" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099604-qbJkjZY2bEaVdPOtT6SQ", "proof_sha256": "2c5cee10180085c1930cfa465ff8df0fb0b07860ef22628ec16c26e21d1383dd" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099700-yyyPltAtypvTurxJaj8X", "proof_sha256": "a41a67e0af5cffd492cbf948764b622413c14a02e56e95549a06babf568b8f4f" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099747-ryHyb9TMw7ozSf52SIq8", "proof_sha256": "106e7ef121d4a6c8a3619b86a6534d1163e63db77ff738a15dd604678edc759b" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099826-oJx2YuEC7xsbjnRYnWsD", "proof_sha256": "d1d8314e107be9f5c168001aca488bfc30616399c3b7be544b110c7ff998fcaa" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099876-MWsnYSTaJhupzsL8LgpY", "proof_sha256": "366a172d98c50a4cf4c7804f1f4c3f5750555012ec101fc85896f9c8e05acf91" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099908-C94cLzp0BkHvWaPw6XcK", "proof_sha256": "b16fb6c0b5c3272aea4e395d31ee23aecbfb8132f0bbc823ef9804d19e8fc7e2" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099950-qZekdYkAm8lEFCyKqLTe", "proof_sha256": "e71cf9be39db9914c477e0ed56bc092602adcf3a7d71d4c362e85cc44ba9417d" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793099997-NjX9yUwVwPXqukq9yUbF", "proof_sha256": "ae09719ccc5b7999ccd7295ce7297c75d02950991fc1163e14e5e404f808ff84" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100048-a6tQk906OeKqEYegbMK2", "proof_sha256": "4514660b9efed0b679fec77306e44a1126e517c171ed3d60afd0897b4ed15728" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100099-bSzHCMIq94BSZLZ7SzS2", "proof_sha256": "113bab2294c75618a501bae9a820ad92a66a58c83db58ff2eaa220245f796a33" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100159-2AGitcGNmOpgMYqf6EUp", "proof_sha256": "9c544c748f84ebb63de41002283a4f01f9c827090585387b4b4c598e22058648" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100214-wie25ilGgYZ9rcOpPCc0", "proof_sha256": "fda879557749e438d7e420b13da506632d3c02ef80e4a043c0905b4ea21b6b5e" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100258-VOEteASwhUMwhcFQRWag", "proof_sha256": "092c8662f124fbdaffff4b8b6f1cd8bb4aee995570e45ee50d9a17fee78dfa23" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100313-UX1VkTZKucnAp5q4FR6v", "proof_sha256": "5682c1a85997cfc42eff053156600dd9377709e552baa765ae0c3d6d6aa71d10" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100359-zZE8KXD3pDvRYwyGWyVg", "proof_sha256": "e93832324623c00099bb30e1fe87002eaf49e78fd99d286312bf47a7c750eb9d" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100405-NwV7nuWBBHD3gTIJPOMa", "proof_sha256": "8fbbe109fa98e99c83006cfdb31666a2a22ec827f23c85738c75c6d43abd9d7d" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100434-poaEAUNonh4sLziyqTKn", "proof_sha256": "fb703c8c80a1cdbbb389e7098e3d33a20af3bb0be2975f43ed04bfaaaa4242ce" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100459-F1iGv2uXHQ6VhVjOIlYu", "proof_sha256": "e21e9a7cac7a2300c84d6e81cab234381cefafd55494ed73b58734c35925d56c" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100480-FWMyk4fhRYlEBrUFCu0Z", "proof_sha256": "632661d413ee0f5e62dd00a4ce1afb076aa0eee9ed34681d151510920433cc5c" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100504-NSf4IrPLfN8TM99kzyOs", "proof_sha256": "109ffae7fc3993dc9bf5b37cbfb6d4678d0264d589139eda35dd38cff0eb369c" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100524-Sd4PxTjxNPsFHcFoSWHW", "proof_sha256": "fd3bb850e0413c92231108e5e1083e26f2552532a516c40c245ead332e71e70b" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100546-6d5DAPeFKjYcCDUMkdBa", "proof_sha256": "cdea9189ba1ea146b2bec2632b3a4a4de812e330e6bb1e0ceb1059182ce4d9a7" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100593-NU7ENOnxL1VjD8ehUfqT", "proof_sha256": "bdc818482bff8ed33d3d1a460d5fd50d8b6b5a146a3f021dca78d638d61d63b7" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100614-MqTnnn4vx96nooYOgxKM", "proof_sha256": "02b81bd85e6e97e6e8d6f56f066dda87250acb29c9e889b72f082f83ca2c5c43" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100626-o3sqfTl7JwfhZvG93asI", "proof_sha256": "23bd39d6258d3946a1b62012e96d5445cfe8458f86d636ad0b346e81af170b5c" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100636-eLsAluUkMkgMZfXJ8nAl", "proof_sha256": "c1d6f8b4a3b24345d7ce36221f0a588a2c35d027a0eb63cb802d63fa2b05592e" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100671-go4NK7ltXFtPZHEZGkVw", "proof_sha256": "2faef1a1dde232a89b5e031d3b07738dc4fa496d422dfd44c468a9f76244ab01" }, { "model": "openai/gpt-5.3-codex", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100696-iWNgpiJe2CDZWRi0vjT2", "proof_sha256": "87fcd2a351dbab5ce88b8c4b46a644a0838e3490e0eeb40c4f4e30a16f595885" }, { "model": "openai/gpt-5.3-codex", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100709-1BWe0fqgGoC9kXRIWVfl", "proof_sha256": "0e794875f3a4caf4f9ff023417672b01bb9e89f9e789b19da954023e592f3388" }, { "model": "openai/gpt-5.3-codex", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100737-U4HI8qkj9bwCmotrDbOQ", "proof_sha256": "2ddb3822c1d01b7a790000e8572cd4186c02fcfea94c27873de6f63a57b77b5e" }, { "model": "openai/gpt-5.3-codex", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100757-bOqzsyl02PeKM550kV8c", "proof_sha256": "f25d55fc00dda3f0b9d58a2adfb564318527088bf1f5b6389f81a4de0e262ae5" }, { "model": "openai/gpt-5.3-codex", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100771-sstt033XGvE311LdWEr9", "proof_sha256": "4375c2414ebadfdeb7cbba97031537786be6dd4ca9c8a47170b3cc14faa46f6f" }, { "model": "openai/gpt-5.3-codex", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100792-cHIbIiQxjMPLTxSL5CPB", "proof_sha256": "197d3647bbd3696189c285e51bf0ecb172969c7d70905cec9ce854c920f1c811" }, { "model": "openai/gpt-5.3-codex", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100807-3a2n3krv16W8aWtAWVGV", "proof_sha256": "f09c9c133ff0d26807212ef74aa38e936615308e80434377160446ef71587e72" }, { "model": "openai/gpt-5.3-codex", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100828-cgCsA5Ew7lizkEDxdUi7", "proof_sha256": "d58a9d6cabd5b69195bb7416e62a2ce768bbbf3d0275216964b3f2b25e1bc76b" }, { "model": "openai/gpt-5.3-codex", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100839-HFUNWA3XOrQ2r2ccPyrC", "proof_sha256": "07106c3357c25f1b72626f0ce68f7d681727ed14a32d0168d72fe09090714eae" }, { "model": "openai/gpt-5.3-codex", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100848-NpR8zJekPrhzc2cA4dQ7", "proof_sha256": "dcd59d1549bd9bcceeccb2a477ee2ff45cc9fe940c077f024831ed5f9c4c4931" }, { "model": "openai/gpt-5.3-codex", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100859-uEjti7zBmkp97B8oLZeh", "proof_sha256": "47602c0e1832e304f5df787e72222afcaeba868fc4d1d11ac83bcfea43ff9821" }, { "model": "openai/gpt-5.3-codex", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100869-HzFg4eQ7w1Z6QObHgTHa", "proof_sha256": "7fffd68a7cd92288b85fb938309d668ce5d7e94e934dbd1f27cee8c1c0a6d55c" }, { "model": "openai/gpt-5.3-codex", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100879-xlqlaSQg1T8GmFZ0TTdH", "proof_sha256": "6d89c1a61f4c537cc8396c2de6c9129c47baf131eb9d027c0234ccae36268c6f" }, { "model": "openai/gpt-5.3-codex", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100888-JfEeRXWxnNJJGMqX9vl9", "proof_sha256": "bd94f44d9200305bc4f27589c253594e56ea24f6e0a2e5e78c5556b8b25d9a5e" }, { "model": "openai/gpt-5.3-codex", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100897-j8u2EKjUwdOoAsaDDDP0", "proof_sha256": "a1fcf14239d5092c2511dda2d8105cc84ae39bcc52fd4a818d9f4f9a1e496b53" }, { "model": "openai/gpt-5.3-codex", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100916-ekD21wMRvptG2oysGrFO", "proof_sha256": "8866938e821ef3c81ce69e4e43beb789bfca1aed5a85d17025279eddcdc881a8" }, { "model": "openai/gpt-5.3-codex", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100926-VPqhFffi5cVLQqOgGvT5", "proof_sha256": "3cb5e086eae4599c454ec09a13ab99d3670b37cbcd1b3088b7dec0c45bba35e2" }, { "model": "openai/gpt-5.3-codex", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100935-m6XlhNItINhA01fHQmw3", "proof_sha256": "b8700a5ea296a2230f49b5c416b4448b51790e56c0bd9c3a486f3646052b456b" }, { "model": "openai/gpt-5.3-codex", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100945-6u7VAaeEkmWGFXPZuWoc", "proof_sha256": "4cc5b0f57a7d63d174c5500cd1480fea374d29ff258bb2426d0baa2c05d75769" }, { "model": "openai/gpt-5.3-codex", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100962-VtYb4znh5UZXhEr5yByo", "proof_sha256": "b9cd85636d9cf5f55311aac2d91a49962776c83ab7a3587380e3c659f31150b3" }, { "model": "openai/gpt-5.3-codex", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100972-O4SQD6xmC32MCrV6Yskr", "proof_sha256": "00f543dee465ba61c5a0fc0e844ed524d475cc13f0206130dd45fa2ffc234e55" }, { "model": "openai/gpt-5.3-codex", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100986-ouYkEvHrg2XN95eqeFHX", "proof_sha256": "bd6bbfa5c1d3d18e2be1b2595c33b5107912f94049ba2c5fdb1f80904691b74d" }, { "model": "openai/gpt-5.3-codex", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793100995-vBEE5RKILLHQJwg7tozL", "proof_sha256": "82b83c86bee997af9254f135b9489c8a49561d28a7c77dbd0f77549c56389074" }, { "model": "openai/gpt-5.3-codex", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793101030-EDWUDFJfbBuWoCBVhld5", "proof_sha256": "1f6daef99efc15a674c15805027b89a18bef935efe50072d351e9424c77c8aae" }, { "model": "openai/gpt-5.3-codex", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793101049-VvcFjcEYH5DbU794cLGv", "proof_sha256": "b183e4c7528208de9b3d09ae9b6b16780ee40015f006e27220f9d81c88555fb3" }, { "model": "openai/gpt-5.3-codex", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793101074-q3dOfdQ5qOWocRgSPxdB", "proof_sha256": "125746305fa51fa1855dac53461f7a943a0ff829dc52447f4f988bad4c8beb19" }, { "model": "openai/gpt-5.3-codex", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793101096-wS9e3rvna3gDQ9yOgtMC", "proof_sha256": "ba7eafe568e96c6903fd614c878c212a3747e5b3a2d16c14cd726cf089df9020" }, { "model": "openai/gpt-5.3-codex", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793101109-D66tFdJRSe6om38nf6vK", "proof_sha256": "7a8ca521e74d141436742ad326b9a3b90a705196e792cf69718f87221771fabb" }, { "model": "openai/gpt-5.3-codex", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793101131-JeE2bwl1VDcczEfT0nin", "proof_sha256": "d53bd4f11046a91924818dd8a13d32a41c203480d6282b2eaaf5b5951fc7e16c" }, { "model": "openai/gpt-5.3-codex", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793101151-6nI5HIZXdZi4pSxVlM8P", "proof_sha256": "577b9d448055235368d681700eb553ec44ee17a499c358bf4626131e5594e71a" }, { "model": "openai/gpt-5.3-codex", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T103825Z", "generation_id": "gen-1793101169-dQyvDNtdLIPcsg7OXuo8", "proof_sha256": "9f3a77bf6c0a7847b75d27ac605d3819503fc5f7a00cccc5205305dc6d7e6026" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793095697-QTpxtdKTN8aZ1nnGnVJE", "proof_sha256": "0de6c873a74d451e7a15b4c5995debd869eda621662741465f131940652dcf24" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793095717-7vvkhFjPDgGqGnigaGwt", "proof_sha256": "a9c1a8bea914df0c41718f7664a1a0acdddf9b633966a55bdf7b6e8714b8f688" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793095737-jfol6f0QxZwPwHLx8A2r", "proof_sha256": "7f1d8f188bfba74b6a10cb6796890ff4231af1b8ca1169a717b60adf2811f208" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793095755-BfC0sfpTkwee344vMtuU", "proof_sha256": "94b640e0524407ba7f76ddae07520823625cb8f6169797852f8889ba6b84784d" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793095770-gAtFiqzn5tUc5bmmgX4w", "proof_sha256": "6387474db58845f53a34ce2518abe19fab6c8579af6f82a6f917db1319e33bd4" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793095783-cf6jdWZojvBL1XQ4N3G5", "proof_sha256": "0e8a4fc964d40e3b6eebd1de4aea9495fd2161ed00451cf3a074dd73f03848e8" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793095804-hpzXt7NB5DM6hbWRvWVi", "proof_sha256": "1f8db3a65f6ae669c36b9b5ec9f07f8f2bd0d7aa6bc48e02d86e6888c6c2d6a0" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793095814-xMkyzRInJ8y20PEetAUf", "proof_sha256": "2056972f44ebb873d0faeb61e43890d80ca4edae8f1cde7d94f3cfe491dc3091" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793095835-6L1lgepggNGiLhAF2fRa", "proof_sha256": "d10edcc7f9f220fda5c52d55d485dba0f76c9da708b3e2d68ebc2044b0fb8657" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "none", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793095845-ym0ouOHTabqpaxdGcMej", "proof_sha256": "ee65043208009b10ac2e2b0518d82a87c5f9964339b6f4be5394ddb0a838a370" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793095856-OkLRjDiBw2HhiZvq2qfe", "proof_sha256": "a1f98684f46d4a59aa8f897e0a98a69ff7b3598cd28ca414016c02e9a764c816" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793095865-XOFq4f4OSCIiEvppoQUD", "proof_sha256": "4fc38abb52ed745bb749b7d35c895d787b2f7de12f90172e27075003854d1c7d" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793095879-d9CY6c0aJS3jJETyfGvm", "proof_sha256": "d1ab6a79e6ef6ab341836049373743631e059f48ead7f0e8aba5caebb55727e6" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793095897-KTH2v9Y9INvTf91vWjNw", "proof_sha256": "67eded04c91682da6414a410e8b2f9566cecce22ee441c8ee77fc8dc73d40cf6" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793095920-WzbkAUZrfh3AAaOGF7VS", "proof_sha256": "403a61253a0c8d68f5aee325e6d73adcbacc33b77168e5f29c34ede5a3fd98ff" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793095943-38TDtHV6C3dRQyJffZSS", "proof_sha256": "80f6938531a491daf9cda093a39f6d85e123cbef2246262e5f4002c21a0838e6" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793095963-6x6SOhL2W4ESK7FDWWSk", "proof_sha256": "05b5eba2c3ea30b35ed68b2a91ba8804a9e24988e6e743706eedd48b2ea95d42" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793095979-2UjBRk0usGGkUKR5Fm1q", "proof_sha256": "189720a2c2ec2e401ec0ff507e73213aba9427ba7bdeb5c0c978a5d898a6bb45" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096000-1NegxXog2gcVxSZF7OFp", "proof_sha256": "7645cafae7e4f306490193fafd6636c0e8e81542060fab8c4780370b3b33ce53" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096011-FpcvOOYPtGVkOEepIYVw", "proof_sha256": "873eb97b0506372264165731825ca547cdf459c3581dee20fd8ccc4ee4bd23b8" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096021-4YJJhql1Vbs12TxGq3Gl", "proof_sha256": "a2b7a8daefa22bb89b1d9409ff0f50a16b95debdd74586d3d43ba788da800bee" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096040-oWbPXdc88aN6ihRR5feM", "proof_sha256": "44a4a7e1e2e5baf75da0fb5ddf5d1200704db5207924bd0df32dce0b72be65c4" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096060-UY7Vn1a5de0x4mggvxcL", "proof_sha256": "fe80583db62c6243fdba813d6297c8aa725e9591ac9096a897f0579b0a4ba3e8" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096100-jsrKClYwkcr1lnsjpX96", "proof_sha256": "553bc830d00e7cd10bd2935050265fed65f09e6656235fb0b615f8ddde7ab80c" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096121-qgxUo2KZgQrdIgz4WbR0", "proof_sha256": "e1078d36dc1075bf109221607dcbbf32909dbe6f63444b05278786b6433b3dce" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096134-rF94CkSnbxl7fqf235vP", "proof_sha256": "76b84608706afd56a56406c26ce4c1cb6ad2b6faefcdfee990128e5c680d834d" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096155-dJhP1PhljRhRaDUPdAzu", "proof_sha256": "890fcc62562e1eadc52ae6b97d5b4ed2a5f2301d1bb19bfccaa556a971a819fa" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "cannot be determined", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096176-TKz94FE4Y8oIzn561XZw", "proof_sha256": "6459f2e4561c9facf1e9d08ae4c274f845d314b708dfae0b0266a4bff2568c8f" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096198-mvavejOqrm9cTHgb9nYV", "proof_sha256": "b375104b2b0cecf238115990392a7ebf82fcf0289d24af9a5dfd808067789b53" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096232-7JkEzHuXksBrOrBUPS05", "proof_sha256": "65a9deeac65b405aacfff65855fcd36722b3cd860ef3fa54fc612d877a6561e3" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096252-snTPOF7iAQrY9WwTGLtj", "proof_sha256": "e4c9f265970e91b38002e6996939b57043ec99765708886ef3697dd6481270cc" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096266-Ihng2VXDEUzpuEJ97Udu", "proof_sha256": "24e70008958502cba765bd276f00e33564d509e46dc836290ed862c836057f34" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096287-nus1KaeqYekYjrU2vP9p", "proof_sha256": "516d924d41bdbcf31df49f528764afee4e16853e783536b7c0cc192087130627" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096308-BBcMQUkbdS6dGMLX0FWB", "proof_sha256": "326c08b0c360c7d18264eb5876b2bc4c9a11ab487f55336a0a5a446f731f3077" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096325-zLizRcdH5OYePhc1lltd", "proof_sha256": "93b1973f9a54a151efea1df2681701e0cc4744ada73d7d25550ebf5440bf6fc0" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096345-V1yWXkoJK1iJHSgXlcPL", "proof_sha256": "0c4b9e8196e36c91f4a1b32389f52f22e50e952e61858e6cafdae8d4376a67fa" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096387-P9C9QraW3DIsTzQ4QoCR", "proof_sha256": "6b1a1da1ab3f21bfdaedfd974343f5272733105cf5204d492e83216ad868df86" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096428-5bLZd05hKXvLEab9hqaj", "proof_sha256": "cc3013c32f82d1e65ef99dec04dffb41cb69d2c7ae088a374a80506129c465a7" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096468-J6f2MzBwTczArgJgt8hF", "proof_sha256": "a4d85e167ceef15b59bb8d6d41cfcdadb0cc93eea75b047b3642dd0ff0f29216" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096509-aPyroh1gN2aFz4IVqdVz", "proof_sha256": "caf6387a016e4991c7abf2e0f524f00b916fd0ecada8c1aa8411c13b292e536a" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096550-MS3L6OC81w8nmtGntRlV", "proof_sha256": "d5de7257f26daadb944b1e7162009ce10f288d9dfcaa63754b31d6344b2808ca" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096582-bISrYbvcbFrgqsns7y52", "proof_sha256": "036576a5283dbd81360106da7ba60a43f9c1e7bc5467dea6ea7de876253acca3" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096624-QqFdKBiB5SCcLA8WQGcF", "proof_sha256": "49289396c0e95485704d98d6907e638a4fe3d96c074628a769c8f0d13a9bde21" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096651-2ukF3G5fP25IPIlplS8B", "proof_sha256": "1b914ebdacbac8ff012784ea1408b3ddc8d0a15bdc86f520f862c07ed889e0fb" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096676-IqRqLyJonC7uZVHgUoPN", "proof_sha256": "80e2652fc6856c29112994bc4ea159d351cad8f93b6d3e1995d20a2a938ddc61" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096696-ZwoODknWLMB0apWUpji0", "proof_sha256": "6b8eff7db4d44cb75742473daaa6d48edebb82d032ea801abfe3ad8d0647c41b" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096717-IwCpOKOiyUKzNpYnnEyx", "proof_sha256": "01bc9515226198b7464314b1bb5abab2c5273a3ebf2937d708e24dc8565359e0" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096735-O83kltZ34ENMCYIDTglY", "proof_sha256": "12511f7cab793ed77a96886d2d349acd792d93389ceca322a15673c518bf5a5d" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096760-z58VvLfB6NTF9FKVS3UT", "proof_sha256": "fdd58adeda26b09def7e292d50e8a443a2dedb06b2da367772f30e670e0a6a74" }, { "model": "openai/gpt-6.1-terra", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096779-RzX0aeKvo5B34Bk4PdXQ", "proof_sha256": "1e73c985f2f13a2942a239a0035ee01d8a86113f25aa8fa704ed5478b513527b" }, { "model": "openai/gpt-6.1-terra", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096797-SToAqgVgfNvNvNXDGAli", "proof_sha256": "9e2fff0632e13488e0db48e1de7b2d91eba2c9750b61edca1770d4caa33cf92e" }, { "model": "openai/gpt-6.1-terra", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096807-A5Tfgybhpb3V2xz8qeZo", "proof_sha256": "941aeb5aac89260147dfee297f4124508f0d47dc80634ee4f1146b41cc0e538d" }, { "model": "openai/gpt-6.1-terra", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096825-DYor6c7AwqnPuoGoVD94", "proof_sha256": "c09a7f0be6ebfd4a08698ae799a835d3a903fc7ee368747d35d60b907e798de7" }, { "model": "openai/gpt-6.1-terra", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096851-CeHyNQuI0XaIiwLP9JX9", "proof_sha256": "f3264c6b51666d2ffd965d4de2e18eee85da9c12ac7958d57ad4a228153f69d5" }, { "model": "openai/gpt-6.1-terra", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096869-cuaLh4Uo5VqYNEaj6D9Y", "proof_sha256": "03b8ccdf287de76adcc1e5f15fa6c9ffdc70492e7d0f07cfd518562be2b1e5e6" }, { "model": "openai/gpt-6.1-terra", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096879-NC3PlZlb4VjHqbyly4b5", "proof_sha256": "9cfc719857430a8abe8eb605c01913d84a5855aab1db1c95d4be6ab2a1f8a433" }, { "model": "openai/gpt-6.1-terra", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096898-BJDq7bMQ1MPDgbo9mDol", "proof_sha256": "1e6b9561ad7fb07094f9aa0a2f4ab0d2bf0f2940ca197e8db7c4d0320d622f83" }, { "model": "openai/gpt-6.1-terra", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096916-2kx6XHLO7ewBMglp5xwr", "proof_sha256": "f75913d5d023ef5b70e220f860aab9f511a518436d30682b33376033848ea357" }, { "model": "openai/gpt-6.1-terra", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096934-OTv2oGp3jU8pmNQ1gTsZ", "proof_sha256": "05fa493ec9393f098f4a871f137dd38a4118a842abb275876e4c3f6de1a03a65" }, { "model": "openai/gpt-6.1-terra", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096952-s4oCeCK8EygxYBizYccR", "proof_sha256": "4e5e1fc5a77f740ada8d5172864323dfc0d5c5172bbfa526fc0f7422dfa51193" }, { "model": "openai/gpt-6.1-terra", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096970-1YSV016s3qPgCpVS9PoU", "proof_sha256": "323ae0050b1b0b0068273f983e4f943ee8a15cc6a16394be230b20009821bc84" }, { "model": "openai/gpt-6.1-terra", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096988-YC4UalrjcSDn72THqvqa", "proof_sha256": "eefeb5b8bf1a03a2db3d9469ba3cedf4263886b63f321fa374296f417b9a23e5" }, { "model": "openai/gpt-6.1-terra", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096996-U45MTE6QJQAXbfkUeKu7", "proof_sha256": "7204a6f78fbd423db431aab5105238e4d39ec84a6dd01dc48f996b62a8483f88" }, { "model": "openai/gpt-6.1-terra", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097015-TsJiWKQxVApRj70astZX", "proof_sha256": "e14d44cc0884c9e9b8add408354449f3163fb8e4991941c340b21b2e5254d7f7" }, { "model": "openai/gpt-6.1-terra", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097024-pVRKAF6aenPejn1JPHcJ", "proof_sha256": "fa123cdd94594af98feba9694c5a6c4da50db7647423f7fc55ba259d3696544d" }, { "model": "openai/gpt-6.1-terra", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097033-c92YQqU2KROkq6Jc4C2X", "proof_sha256": "d740a906c3673268db6d5113884cc6ed93afc7192925c8e97ff87c440bda2f58" }, { "model": "openai/gpt-6.1-terra", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097052-CzMSakctQbV3JEG50gf6", "proof_sha256": "786ce166b897f117a1a6f2fabdf1d68e9f29375c5a2632aeb2f13f46ecf9f462" }, { "model": "openai/gpt-6.1-terra", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097070-b8IqpkP6b0q608NcVMbL", "proof_sha256": "ff9f25e2b7e99bb6a2b89bb960a58a88de9e6c17c78baf8ff63d8a93fcf15e00" }, { "model": "openai/gpt-6.1-terra", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097080-neOzUx6abikE0WEmKdqI", "proof_sha256": "59d87a2802fee7f2a084c8a348cac621bb14a818e5f1d6a899054305e742d53c" }, { "model": "openai/gpt-6.1-terra", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097098-gk7MlhbgrSB6ixpaT2ZC", "proof_sha256": "063e6b7826dc945810aa217045c0ca05881305d39e0b17936bf28f25b62fa78b" }, { "model": "openai/gpt-6.1-terra", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097107-oqETR1iNSqsUFDYxcBKS", "proof_sha256": "6b11dc8f9a474e20a21a9f5bd1918ec2e42a4478f28f679e96cccd9dbc06547e" }, { "model": "openai/gpt-6.1-terra", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097116-GbWTrvlBrQrlyRw7kAHq", "proof_sha256": "45794c5b506c54979b7ebb22fd8ee7d6862bc5d40625abce7c179d67bb2c9fad" }, { "model": "openai/gpt-6.1-terra", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097128-XiyZ60lWiR3QfgsOJMok", "proof_sha256": "61ae8207681fad8ab1065d5f14dfdae2e7c8e0c4fe84c244aa9aaf7708451569" }, { "model": "openai/gpt-6.1-terra", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097138-hxXBxUfi0XRygyfwR1Fi", "proof_sha256": "cd384c1b1cc0e55846ec1a73144cb8edae5089f6d5d16d5ecee781ed42e3a6e1" }, { "model": "openai/gpt-6.1-terra", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097158-VvGh6KwVJouPfBg1H1NG", "proof_sha256": "2c754bf5c6b5d58794959d409c02a7f0f7c772e0932c90765f6bab9847bc7999" }, { "model": "openai/gpt-6.1-terra", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097186-uJX8pFg3Vx7ayxOF8Jl7", "proof_sha256": "68e84895b3229e5abdafe4f46df041a1f88ea8a175aa1cb20f9ed4f21b92f94a" }, { "model": "openai/gpt-6.1-terra", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097196-IilWfsniYwjW1CpXkiSR", "proof_sha256": "034d0c9d03b1e40987bd28ac49c7ae1026466d46bfb31edb99c0c6a0a00400f7" }, { "model": "openai/gpt-6.1-terra", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097217-6kSkcJ2XKzydhYHws4PB", "proof_sha256": "79271052f7ef3073ca2a2cee6ee502510a4a8a3025256d01f9955d7933052ed8" }, { "model": "openai/gpt-6.1-terra", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097226-pCLdlCNdrngYvOHIM4rW", "proof_sha256": "b10d061430f023a4800e2c1b1c6ee6c75cb13f337d4daf5a4f89c9731a0416b2" }, { "model": "openai/gpt-6.1-terra", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097244-V9xNBynr6TccFBeJhq6J", "proof_sha256": "042736170220b76833dff039dae789ef8ffca6774ed38b017c4f821720dccce4" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793095698-TtjnOuhbUIbgyJazlrNn", "proof_sha256": "e8bb3611b12b42aab39eabe190684cc923c104579c97c4289a4f3abef715be7d" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793095725-Z4VcjmUX37KkTg6YC0Hr", "proof_sha256": "49351b82eacbf42877dbf3e692b40309c160e217eecdc8dc947eec5dbc4dea32" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793095747-BrEtXMvweiudYiQ01nFC", "proof_sha256": "0efe49a08f78304bc2501bab7a962b5960d0187c62c30c13657c74d7907c5d02" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793095767-0xrj4btMRiSowMFoXT4m", "proof_sha256": "c6860ab1104441c44f51139b9f2502526501d29d47b3af09da6347543785fe57" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793095792-RKIBqcm74uiT0vqk9OKe", "proof_sha256": "8880c103732da7ec2ba771cc649247721ad0ebd1df90ac67786256b4c4cf4baf" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793095818-JXnFsrO55Ey45DdAzewl", "proof_sha256": "66051840d7e22e7af65434b135912f6094f3ce57bcbf0239c7162e1fd68e3045" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793095835-ZZWBP9Rt4NKM1rxSkrlk", "proof_sha256": "af86f82b5a69e8673459c1b62ae7b2fbb9f05b9bcb9843d25a8d0c22588aa954" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793095848-fvJPW7ZmPm9wsSnPXhMC", "proof_sha256": "3ffda32a68ce9b70464e91b8139c36e2254fca08cc5d4db538c6b5cfdc78e231" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793095870-ccspMf9MOLJ0uNvoXR26", "proof_sha256": "f86e68d5b2e8c6e844db8b915bfd8cb23c807018b721fb1fcd458a51abdb37f1" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "no valid option", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793095882-ZaZ4zgLQOk8NpTyz35bd", "proof_sha256": "ec156ac0ebc3786e88b6436275a5f5777d82bbe93de29edb96aafe48c4b9ba6c" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793095905-5IVt8c1NgszarMAEObrk", "proof_sha256": "d763bf059a8f60c4f5854cd23d18b94be201ad24c9d560f0e45a55e372cbc232" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793095925-l0KDpSeAqyCadyn1zXpe", "proof_sha256": "e69b4d97dfbe6ed49837c98a639914eae91f14c7120fddabe675faacf5ce7007" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793095953-cm3E13hRgk4fkmbVkUkV", "proof_sha256": "1e615d8c8e7caca21e85e38122c2508212fe0ebd4fc51a0719cc603c777a38ff" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793095971-QD14iUR8pxBxUJ3IT5F7", "proof_sha256": "23e566a254d86b5b10ef7255b08008fdfcae48624ba9d80d07bbab9a09447e27" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793095983-gVCWR6rJhY0Majuksohu", "proof_sha256": "48d20df91204cd4cd27e306adabec0b9c3a7b8ef4b755c69fbd7a689d23113e3" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096013-IQYUPQuptaQytNsZHhQf", "proof_sha256": "8b662141bdaf6a84248dd39bba7e75ffabb0adbc77e2dd9a2756ce8ac4a0b1dc" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096035-pWecY2WNE37FAWAnNWAI", "proof_sha256": "04c7596d4240be28be33d7ff75084683f42dbc422958169b919f8f078c0420f9" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096047-hiVPFOXCCQTmoLs27TEH", "proof_sha256": "3c4b4af75829d9242b07ec72bc29067f44b8973886981b113cab4a8ccc3551e3" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096060-IWf2svhCcMeCUPmryi8M", "proof_sha256": "5ef62d4e959c8a64e47a25bdeda404e924fda9d63fa121eb7037123c49de3c01" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096105-7bN6nLc4SvfNoLJt9D92", "proof_sha256": "a8ee3b73ddd68fc8488f66223fa877bbf13d3e0cd4236041cc5a1c7c7462225a" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096129-hiWDnnrR5USEeDhtYmff", "proof_sha256": "6671908fa970521e5b814f2758a7cc24cc26e257d780266d59a42f4578bef6e0" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096151-1MTtthlUB2XUUEyHkXDJ", "proof_sha256": "8ad138d23f59c13d7b96b7e8e98a8380f642ea8a94846f140517198bdb8fec86" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096188-aWun2pzMYeZMcaUZuzhV", "proof_sha256": "558c89c1baac3a07c6424eb203399eabf9492ca3478fe5264677c69e03db5602" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096233-yuVL7QYYkLlswC0ydAEo", "proof_sha256": "51098cff6051bfb3c0679d4caee72148f947ce252c0753912d21dd62f13e0251" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096248-2NeZcg00ZkmvpDL10R3V", "proof_sha256": "2173ca2da50f613478abc7ece02156605887271cf46cd042bbcd80e13513866d" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096264-bkkOCpF0odumMVu9r89R", "proof_sha256": "904a5265c6abcd17226135dccabc8b4f405a85fd7830ee7927bddae84c68a697" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096289-vwFH7fu27PcEPPMIfZnU", "proof_sha256": "54c7f2807fd94ab0f924a390db50c65ce904ea9521855e482d28e6b101171b5e" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "indeterminate", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096317-WHqCNm8FMi4NlWuTLKFw", "proof_sha256": "9504c28b61a0831bf6422821afd3d03f62f4ba35a8a84ebc56e8a816ff0d1f38" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096342-s25szghsIBG95lMlADgv", "proof_sha256": "fb6896eca08017306dc47769d2e10071a9f1b02e73aee16dc17cdd5aae908b6e" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096387-JniSW92HJQ9N9OVRyRoG", "proof_sha256": "84d6f041d0677a56e3c0fdd3207b184a0eac60d8cbb00938bdb1f349ae8ad749" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096431-WKhSNzHryJ0s8STqlDEa", "proof_sha256": "af26beab7db9827811cdafd1deb1add097261d65157fd21b3e43fb263677ba25" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096479-hr9Dq7I8DVDJ9pm5xc9n", "proof_sha256": "83f81e977cffe066fc92de4e82622176539d91c3f3ea6d4b0ab2d29be7554a4f" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096522-jIGi1CO39TP5GooUWB5e", "proof_sha256": "eb244c337bbf010f6aecc6b79a43aa0338dc2a3141d3b30de8ad01fd109dc360" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096558-PrsqjwMgwuhqd3mojtdn", "proof_sha256": "648101da799b5573c1df1471add5cff87420d570c3d74ddfa260768d65de0ae8" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096604-ldLpkmBsHaqDNZ284B4W", "proof_sha256": "ed88719eeb4adb5153a82b58b2234d4b27d364fbfbc73a1507d91b65169fb163" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096647-y6aP073Mye1u1Ie5jfUm", "proof_sha256": "910ebcd1789fc873e4f2da0a046ad78de5eb32f6406d6eeebd6a2aaad7bfeab6" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096669-hKnzPoBbxv8E4r4uWRUc", "proof_sha256": "6e48582d5cbb1ac8db45f8464dc3fc7b8b7cff51aefa385adb2573dae5274add" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096696-LSVXXecGjXPnTmS9yWv0", "proof_sha256": "31f2b9684af80c18b73bcc9eefef5ace8a29fe3bcde9d908f41003174a3f24ca" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096717-kNC5NYxEBj6ReYjwtunv", "proof_sha256": "911379a7845347cd6ea0515c5511f332209d3beee3cb89f7f7244ffb112d665c" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096739-PXsJnz5tWwWEX6CUFwEX", "proof_sha256": "1ef873447fc2cd37b9c4d953a3da0aa2d0a57ce3a2ec4f29c1ce1ff51e4c38b8" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096760-u8rYZNqBFoT64q8JXUpV", "proof_sha256": "ed0fc8a3122d6367bb619ace93dfae9eea7db5adbcb13079cff0f2de71674b9e" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096771-obxaZrQylz3OjccgTZ3H", "proof_sha256": "f0950dbc0a5758c32e91380d2545f86ac99fbd27ae36aa96179df531a3f8fd0b" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096793-jIxYqTR5ddQgJTijRfea", "proof_sha256": "d1b672f0c7cf0e3835d3b13d6c6594abac563792e3af740983b0cebb12eef398" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096814-a0NNqDqqTRV1LuM0isig", "proof_sha256": "85e55a01fcd95b56b95e315905f3e56f557f372c2c3e77aa4c9ea86e3d0ec661" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096848-xU94cqkzpDQYG48mF6Ow", "proof_sha256": "f2bb83405d0c3522746b4d295b05c5422ba8594a86044ffbb4ec3df08b970b79" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096869-OEPPS0wEkmEWQGkl3YUx", "proof_sha256": "0fcc98bad41435de3b18f9528b9c5bdce3cf6b0c5ad34477801b13dc3a514681" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096882-nMD4vus48yFbkcyKusB1", "proof_sha256": "ad631671b10b1a732802cb06dd193aaaf88922ce6e51ea70d96d686f2f2cdce7" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096892-zjlPhcMFXsFglEJCRpiJ", "proof_sha256": "d3819108cf782d8e5e817032636ca1a1fa43f77c4b1586d4b973f83e5541dd74" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096908-aL918bz9r18Gl3jgJCZz", "proof_sha256": "0fa20f7007a5a26f3e20bbfb54c2393ad1296a46b034230d835a6473c4db9220" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096919-Wq4Wqbi3W604T2r9LX6h", "proof_sha256": "24224c80cbd0b22dd5d3d2436a0bbb9e40187ef4f91639e781fc69ffe59f67bf" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096938-yZG7TP2hJuKTxIc2ZdUN", "proof_sha256": "a031c690b4a95079b2f7a23abe41f6444726a5850b1c2f2bdb6339ff67d67566" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096960-nPXxa7Pv6pGk4vAvCiiz", "proof_sha256": "a898f33d3dd0ed11537cdeaffac4cac4ebec82186f45eb8e876899c086d42364" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096972-M0VjVXxSxNY6ocObBhec", "proof_sha256": "166b2f7f0d1aa27c68ae62da45f94d5cee7ce5fc70553e348cc1bb03f192bd1a" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793096983-wTmrxsyyeDTiCb2lExMk", "proof_sha256": "4e6f4b31de1558575ddc76a19e278d6635b295c1d3b6f73398a13dccf1398811" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097005-xWJ6kvBfUOi9OkCHccm2", "proof_sha256": "8fdcaff7a65fce83e6f50342ec27fbc09dbf276311cf00236edce023941ca9c1" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097028-h3Uiy1lJZi6ZcPz0sm2J", "proof_sha256": "f15e0775fc8c52d8f2f4f29f3cb1ad43aae3383274c5a9098132e59069bfbe21" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097050-pEc36lvVahYAEfDvELpD", "proof_sha256": "3b6389f0ff166d41b6cf96a465d8d09e369bb678559aaf5da5a20abcc0ab016d" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097062-FkQcXLk31Y2Bmep2w0nu", "proof_sha256": "1bdf0c91b12d0e02194ca981ec296816d25aacf80c674fed171d89e18d31b01c" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097072-KrR8zTgmXkB2OElBowel", "proof_sha256": "70bba33aede951e5e6190bc212234fa56f235c04f73dd4907cb3be80c077a12f" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097083-cTHWRmRVK5FLOd63zk4C", "proof_sha256": "8ab6e7254e53042ac95e27cb826c8e05ef9518f7fffd95ae2162025ed2d671bf" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097095-LFtgChGgErkjLem6wJlB", "proof_sha256": "bee7dba7861a3be214b05c601476c960b3a47b6ada7ce41cc6e9ff39cba7ef79" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097115-vNg5a7a9HBbI5auwporV", "proof_sha256": "8b1a420273bc8ea43779304050e12a8bf3f98c701f14d0be625b1751992ee623" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097135-kHBSPH5V3fDY8118bGIO", "proof_sha256": "10a3ffaefb8124e9a015585acf82ef227a026ec7213bcb426ada3723eb3dc232" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097158-cXJWmTJ8bIQR3hjInnRV", "proof_sha256": "e0ab8a575651e284ce20ed3ebbd1b45fc06398e770fbd7c4c0d9af7e695c702f" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097169-PAWrlRhHEXmpQ0YjH7G0", "proof_sha256": "a5740cdd384578089a02da7857bec4c191cf8f2849fb25738678ecd9370e4e89" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097186-lsXKgAokETvKZDqYHvu1", "proof_sha256": "aa88002882e52c42212627d7138d97a369a70d657334c95bbaac86c89194d763" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097197-s05cSeucX98xOGPZOKv7", "proof_sha256": "973b7490cb1884372ea8b00ab58573a571a41f82f4a0ad4a186ec8097c27ffcd" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097217-NsDjmXhj1eq0l66xfbez", "proof_sha256": "6a7d892114fd717fb12a0e518cab495db1c20f6432df925d0bbef5adb3b4a72d" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097241-7FkOpYGm8IiMoAQ9gvWs", "proof_sha256": "deba06c18aadae52a00b49335abdda7e3b5f026057634f41b0530a4c3292c118" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097251-pxbgJyhZybboPXdxQHWz", "proof_sha256": "ce4452792dbbd63c56cdcdc5bcc55fa1141ec5a0f5ef840694e7f5658078a412" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097260-Kn2MRFSBuAhNc6OXj8hR", "proof_sha256": "b25214f90d82386b2ceb5b76482096e55411a5d5000ba2d50812153f2597b5a0" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097280-jsCn5lFq6MIyB99JWHm2", "proof_sha256": "8e53f0e7e2a85d3177a36fa300f1af1282e50d42b7f5b8dcd15e392d1826102e" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097303-sJLbZCCrSpT7b7I8P7yn", "proof_sha256": "2fdbef18e4e138583f399a84beffb3504e98be2d09a78d93624c0b8e8f74bae5" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097323-N9V0pxPYr2IyoYuEXRgi", "proof_sha256": "3b2c0507442c82094fbac17783911b0d0b169c0d7a71c49c6c74d3344e68f824" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097346-9n4mRJz0eDLKaE33qTeG", "proof_sha256": "d7375d399b57bf5b2bd8daab782238c9b0a747ccfbde918b0e0dde14bf161b95" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097367-0CFXhg4Fp7sTwKXeZpyn", "proof_sha256": "c3866e5da18b3ecbb4fa87a9a530db96b6cad9aaa8c63571c07b17459981c3da" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097388-eYExbgXHw5BpVF6vpPo5", "proof_sha256": "2db7465b16ea1b40c77f859c6b2e3ada8563e534b7e1e9bdbaf75024cccf3fa5" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097401-E0HU1sAJV8FFldDQpwwl", "proof_sha256": "6ae594956a2bf7e9aae898feb9fad4dad5a9dfc38271c02aacdfbda29b79576e" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097416-WlXTVtnJo0qKxKXWgyVN", "proof_sha256": "d7a47e93e84ec1698ec87776e092de22c3e3a664d09fdaf0407beb60b4679637" }, { "model": "openai/gpt-6.1-terra-pro", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261027T100730Z", "generation_id": "gen-1793097433-khiMXNTPJ7OfiBSbo2X4", "proof_sha256": "086c38165c2ec3944f6983296834a7160ff99265ab53bb20d0eced9e247a8567" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Darkbloom", "run_id": "20261025T200000Z", "generation_id": "gen-1792954184-vLImOtd6yv2hbW7tNbhX", "proof_sha256": "e4cda34f94627b115ed5e40b5dc810d535258ed06e3a263c837608e2414d711d" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Darkbloom", "run_id": "20261025T200000Z", "generation_id": "gen-1792954198-lLya5hYXMuc0N6tTz0St", "proof_sha256": "968659746027f76c3205e313a8bf85cba07fcd3054cd8b6ce8b23a62a8b6145c" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DekaLLM", "run_id": "20261025T200000Z", "generation_id": "gen-1792954256-CXyYPnzPgp6xVGMTndHt", "proof_sha256": "b65059e8f22099591d860f9857cfdd9d0a1717281339eb4b550a534a1866c35c" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Darkbloom", "run_id": "20261025T200000Z", "generation_id": "gen-1792954282-Zs6wDFDhBHYtOlwxBm26", "proof_sha256": "82ec6d101dddf45b13bc20bf47054b016da65f3931b7e72fc8fb87d622582edd" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792954308-qbTgko9nZ7gAqsbG5vUx", "proof_sha256": "ace3b8c5421ffb60f07d62f869d815f7f77b90c4b2256f492973979ed25d5574" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "\\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792954327-SpjYb8rtQP2sEDk5gcEg", "proof_sha256": "f94871511211d9f3fa3e067cfc2031ea437b8f16d694e71251ea66b9b6a14187" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954345-bMXRW22nGcRRBecdRHjW", "proof_sha256": "3e9e226a89bc349fc2c6998415ae0164675a91fb85e80fd443fc9af68390dd00" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792954354-ryBhyW5G72Z80EFJyWQX", "proof_sha256": "078f7f93f3826a96490d40ea3539758623d2c713a62c7aa48cf22b5d208ff0a0" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792954369-64VimCqgqetcUidzRpPO", "proof_sha256": "bb5c0447eb91f0d5d6b994019f439926e91d73fd6c98a598d2421d9a774db358" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Darkbloom", "run_id": "20261025T200000Z", "generation_id": "gen-1792954378-iuM8P47OpFnoMJNAKGAH", "proof_sha256": "1010232011759f37f445d4fc10ce0b5aaa52b26961139fb6f960f0cebdc59f20" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Darkbloom", "run_id": "20261025T200000Z", "generation_id": "gen-1792954390-Sqe5L5Yl2eAqSpZoMo1F", "proof_sha256": "3888b664c71f32f9e35ace75e78eaf990954400adeab8c6887c1278a928e9e27" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792954405-wWeZQBrKDCp6zP7KNFQK", "proof_sha256": "9f7b69436de0f863be0c817d5e8e27587e42a2f68265e727a512d498f7486830" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792954418-Qz6dlhNlmLAK3uLGoMXf", "proof_sha256": "81fa2210f36cce2ccce2b6005e0500c060e7f2b75363c124769034b3b9c75296" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792954428-7OtzTTInvGUaBMZNOUqt", "proof_sha256": "f38717345eb65b2af108de5eb87ebab2a0a1a49954b676509a1b31bdb56b1615" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Darkbloom", "run_id": "20261025T200000Z", "generation_id": "gen-1792954439-N26tYGUJ2SNtmGs21Q5P", "proof_sha256": "1befc0b0ce628b70d46e911530fcbe264551a5498daae63e870bc0d7624e6556" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "which corresponds to option g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Darkbloom", "run_id": "20261025T200000Z", "generation_id": "gen-1792954459-a564EMIAAXzgKGACb4ep", "proof_sha256": "efc3cdfc46cfb978dc14f3fa610ec5322fc5536b7ca1a8f0658e42f87981c378" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Darkbloom", "run_id": "20261025T200000Z", "generation_id": "gen-1792954479-zw8CoWkSnUVNXlPVyTRb", "proof_sha256": "2c8bce4dfa487c7c51aa07b281584572ad9dcaada63dc7e55eb5f73f127a962e" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792954487-bpjCJ87YFts6dufQBiwA", "proof_sha256": "48adeb794f73a0d65b23b64a1590cb09185f4127602c3026637e0af6978870c2" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "\\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Darkbloom", "run_id": "20261025T200000Z", "generation_id": "gen-1792954499-48ThNUXoqZdyBGboXufa", "proof_sha256": "b93a5cfaf2c9a7537c0301d32c3f06c97ff3907b34a5edeb9bdce21e10a5e2dc" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792954516-nGeJQr2wnb59QYR4W7eW", "proof_sha256": "202ca11db9c8451dcf472b616a2014c0985fadfec299bb83a49864aa6311cee6" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Darkbloom", "run_id": "20261025T200000Z", "generation_id": "gen-1792954528-zSuJ6ddaKAuy2aPN7cbF", "proof_sha256": "5815d5749e6346ac646d0e9e166a63b0daf241190391c5ea36ee5110f98d7a2d" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "no correct option provided)", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792954546-g3i3FWP2vWt38x0ABJgj", "proof_sha256": "7a161b8ab72531ba3be6f0153bbc2043257100548efbd971859e6fb89b6ba91d" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792954570-sOhWXqzkXZuqbUbsv9qx", "proof_sha256": "4d2f9c9ce34171c0354430fd4218d2e1abc6dff44b8a62354f288676db3322d5" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "SiliconFlow", "run_id": "20261025T200000Z", "generation_id": "gen-1792954580-YmV1ryhlDXjjr7E9LlFc", "proof_sha256": "e5bfa00acf14863a187c57f3a0eb0a07e3b6100fe7a4b44185ef7c98acfdca90" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "none of the above options match the calculated boundary‑layer thicknesses", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792954597-f8DrAoxt1X8LniPTdSA8", "proof_sha256": "3efbcb265cf55017d5171835a4f23dcba02512368527248cea29200e07e5dfec" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Darkbloom", "run_id": "20261025T200000Z", "generation_id": "gen-1792954607-HEe9bBhSLnnQlThRdlKO", "proof_sha256": "39623a4a35c847333c78766b9c8fa7c1f5a3a02a8cdd2b5aa6b6ececc91d84c0" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954620-zror8lpcXCz0UBAn8TQh", "proof_sha256": "65c53340464adfd26df414a465b215f5d85e9623abe535605fd2b9aec865d1db" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792954629-vTyu7sz6wp5s8YG91Mp0", "proof_sha256": "b79910bab87656e59b3a77817f8eb18292e114ee239d2d539fcb67167183407c" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792954637-ebvfkPPmF7F3WOAaOo7s", "proof_sha256": "d600a2174a9cbeef33f7f1eb253a7fc245bfe3a734c0eac99b1774c7322dd1f7" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "SiliconFlow", "run_id": "20261025T200000Z", "generation_id": "gen-1792954654-ABEJJk7G8p6vP7iEakMW", "proof_sha256": "223580744819ceca93e0ec9fb1707d7b5dbd45cb3995f9e25f1ba2653f3560a4" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DekaLLM", "run_id": "20261025T200000Z", "generation_id": "gen-1792954679-k2t90DkNKKtnztGyzbbv", "proof_sha256": "64f605f34a1b994b6cf278f39180a5176c033390c635372b1cf360f902bf2a7e" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792955249-DhUoOR0MiqJYVGOnD47w", "proof_sha256": "389a336a694b3bd20b7a756b4b883dd2c7ff3d724912319b6ca79337444f5fdb" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Darkbloom", "run_id": "20261025T200000Z", "generation_id": "gen-1792955268-Aa7qyHy2TB3Hrg0rNp1k", "proof_sha256": "d465d7e32eb9c99dd24b6c048c436beb8122d63660c5fd74cc011d67f07586d5" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792955286-bMW4kmqkqE2N51LWsWPr", "proof_sha256": "a9736c9c448e75989ef618f0a5538d1f286af4420092011be6ec7d63a0b27e1b" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Darkbloom", "run_id": "20261025T200000Z", "generation_id": "gen-1792955301-M3N3KZGO1EmmakkXrY5S", "proof_sha256": "736ab2dbd2991dfb79213a0f06374aaa4b8e9b26a33ca1fcba4a43f33813bd3a" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792955323-6TmPNT6Lg3VtBkoaDsyt", "proof_sha256": "71027a0f087489ae2c640e057c64e7446f773a1ef75eb9d882a21b06b0dabeb5" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792955336-7qHsnlcbtlgNGd1A0NW6", "proof_sha256": "1f57c22f382518942f8648fb4b515ab30d580e851fc5fd939428754eed3867b2" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792955374-DRZYA9ECbaanzcCltKqD", "proof_sha256": "5a9df33d17a561a7bdf572d6032d0b5890417b83671af19abfdfee0619013ef0" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792955414-jOzwZj3YIbMSdydhPVke", "proof_sha256": "cb367fdbf2caac292e0e29422dda7d5abf0618f06ff81c01fc16545f1ea72a58" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792955452-6UEfGpMq4V1s58Ilpd0Y", "proof_sha256": "1437d69fefacd40027c1cd8b46a645550c2d13546172c49e784869cd788408e0" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792955491-EiTlnFDJxTA2dNsCSkoS", "proof_sha256": "f0ad7dca507d7ff65a8bd267b2fba5e2dd16a0ffc3d199fd14c3141868a6cbd4" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Darkbloom", "run_id": "20261025T200000Z", "generation_id": "gen-1792955522-vTjaTc3un8Djm2nPHkTb", "proof_sha256": "9425bb0f309f146ccd00758a37aedfcf1aaad5d521beed6bf9434c80efc8bd26" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Darkbloom", "run_id": "20261025T200000Z", "generation_id": "gen-1792955564-1r5lqKENVeSOfs5IJPXO", "proof_sha256": "2d8a01ef7384d9c19c8f84c7a5cbcbc9663c0f54dd51428a627345d0e349d81c" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792955605-Di3J0D39r2tkA0ZO91sH", "proof_sha256": "2a2c06fab19ca85c63ae07979056b32cd2c850da69f6f405244e6deacc337bd9" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "SiliconFlow", "run_id": "20261025T200000Z", "generation_id": "gen-1792955648-zc2Y8wdL8shopGTZYeDE", "proof_sha256": "c16c03d33dc3bdd6b7c4a39163ebe32f2cd5c3f6df8f1851144478e3b73918e8" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792955680-cu4LLmwEWuFlsWFKbbRV", "proof_sha256": "134fc4ab120afe6bf4b1cef128ccf54fd2a388ad5c5e8881c937c8107566cbf5" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Darkbloom", "run_id": "20261025T200000Z", "generation_id": "gen-1792955797-hOLrlLWJwJfZ98Wo3v9L", "proof_sha256": "d832c7f3d450bbf472faf1c7915ff416dbe2eb68326226cf4c7b7179fcaf9f22" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792955803-1Bzs1Oy7EKMTnk1ITbkO", "proof_sha256": "32b8e41451d11fa4eba07a84db1609eeb5d92db820dec7783f43dd981d0eac4f" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Darkbloom", "run_id": "20261025T200000Z", "generation_id": "gen-1792955850-pwmHQeaOmgU0fAkThb53", "proof_sha256": "75f288159adb5b3ff00c324c406cf7ffb820c7bfb41d956e10c6797f44dc904e" }, { "model": "openai/gpt-oss-20b", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792955902-pvmpzenvUWxeuniBkvv3", "proof_sha256": "1f127d9d66f9e094e5e19d6725e38991964e1b0ab3dc2bbd13931ff017aa3ee2" }, { "model": "openai/gpt-oss-20b", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Darkbloom", "run_id": "20261025T200000Z", "generation_id": "gen-1792955943-Umpd8UwRaxL5QB1IgcsV", "proof_sha256": "db785c7cbb45707d1c5e069cfa2deaaacb20c16d367003b43769960a1414eb10" }, { "model": "openai/gpt-oss-20b", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Darkbloom", "run_id": "20261025T200000Z", "generation_id": "gen-1792955984-1j1UAS0UpccOHxIsWayJ", "proof_sha256": "63a74b58ef5125d8e5bcbda3dbe8818948dee23ed33016c62a1e44b0ecbea6ff" }, { "model": "openai/gpt-oss-20b", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DekaLLM", "run_id": "20261025T200000Z", "generation_id": "gen-1792955993-lzvHd0u1gNCEk66oQWLh", "proof_sha256": "bcbdf9bc3c9e8309b0e7b90b423ec14f89f3bcab062316dff4594dc083c32182" }, { "model": "openai/gpt-oss-20b", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792956005-EeP7VGg8lYiAl1XnNGFJ", "proof_sha256": "5fff3fc5109cc5ccfd7d908399668021193c5fcb34f7077a0d098edc7a58ab3e" }, { "model": "openai/gpt-oss-20b", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956014-KO2J5LSkM8cdMuJrxbOB", "proof_sha256": "861c4b2a17f8a7cc05543c502330aae7fef038bc5b5fc0cce1541e3b5de2b894" }, { "model": "openai/gpt-oss-20b", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Darkbloom", "run_id": "20261025T200000Z", "generation_id": "gen-1792956025-Z4stRY7U0TyA02i5mzSS", "proof_sha256": "3c77e62012efc6d6db2a9a06aea846d28ecd448ae85baa305156589987c31622" }, { "model": "openai/gpt-oss-20b", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792956033-NWux6HvH7tmpA9TK8s9e", "proof_sha256": "9ca299f9ca252a120d601db43ebd42786fc8d0b8cbc6da7c9e4a2c6fc5d9ae83" }, { "model": "openai/gpt-oss-20b", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "SiliconFlow", "run_id": "20261025T200000Z", "generation_id": "gen-1792956041-muFP12KyLaEsoo2JQL9r", "proof_sha256": "16f931ba3b1ece36fbffa80c3507a62de200247aca36a1ba1c2e6e53bde60247" }, { "model": "openai/gpt-oss-20b", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792956061-pN3Oo4AA86cxy4lMnLGa", "proof_sha256": "fabecea8c9cbce977fbdb1dbb112a436366c3cd993daa412ff6072944fbe92f8" }, { "model": "openai/gpt-oss-20b", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Groq", "run_id": "20261025T200000Z", "generation_id": "gen-1792956069-0a9CMB5y7atC4QM0XlLY", "proof_sha256": "9e505214905b06a1fa8a33128cb78443d5756b73001ddf012d19eca42b47361a" }, { "model": "openai/gpt-oss-20b", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792956076-kMo3A4EDe4U24Spck81z", "proof_sha256": "a0d77dc39aa0865d800270e0ae26e871a70d72139b2d83f71af23ce1125a44ca" }, { "model": "openai/gpt-oss-20b", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956084-O2jTDyp2CwiMzxia3PeQ", "proof_sha256": "c535371a25099e31f1c5986b7376c707acf1b035a67160b5df29be2de49aec99" }, { "model": "openai/gpt-oss-20b", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956093-v2f9oWnLOpqWpb8ZzWEb", "proof_sha256": "0773f8d68b9417ecab81efe26878dc745d541fe4bb0cdf4fab554711e004a910" }, { "model": "openai/gpt-oss-20b", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792956104-JnHPhqRxPs7nQ7LcXhxB", "proof_sha256": "42b1b17e748b9e9631ff3bce623715c435c5449a9442880d2eff4c227236ce5a" }, { "model": "openai/gpt-oss-20b", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956112-q2ihTySURHenCbnOD7VY", "proof_sha256": "fc448d9895361e606fdfda3377c81efbee6c71f84ba2f66d5b673f95b86dd01e" }, { "model": "openai/gpt-oss-20b", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Darkbloom", "run_id": "20261025T200000Z", "generation_id": "gen-1792956122-shShWoSUOJyydMe3OAzE", "proof_sha256": "ae729107f0bdf768d64dbf9dbd02aed59d2222b62c4a400b8a253067f3a03549" }, { "model": "openai/gpt-oss-20b", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Darkbloom", "run_id": "20261025T200000Z", "generation_id": "gen-1792956130-vz05GnF0AFTMcWLyvkC8", "proof_sha256": "556bcf985066d1884147f68dc4fd77ca1ef627c7129e619e393af6b1db5dbed5" }, { "model": "openai/gpt-oss-20b", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792956138-575hDF0fimiXTrjaBUS6", "proof_sha256": "84f90d6272efce6c36a41114ffcb9522b40ac8a82e62c234d4fe345d9042b18b" }, { "model": "openai/gpt-oss-20b", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DekaLLM", "run_id": "20261025T200000Z", "generation_id": "gen-1792956147-4UcFe2qQEfvPHbFApMjt", "proof_sha256": "2e23110b5082f8ca29de5825ec020c808007980f31e6065f8c7f973bba5d729d" }, { "model": "openai/gpt-oss-20b", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Darkbloom", "run_id": "20261025T200000Z", "generation_id": "gen-1792956156-3Sj3YNugl8XPBLnGwx8R", "proof_sha256": "a88c8434e30c29ec028db41286b3bd84fb3e45a32c46a0c3fc7f2106b3857c11" }, { "model": "openai/gpt-oss-20b", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "SiliconFlow", "run_id": "20261025T200000Z", "generation_id": "gen-1792956164-R3N0zzUEVMtDTAW2sjIP", "proof_sha256": "eac63b82becf7e58c1e3e45113a38dd1d48e84a050852050130864b0a224df1d" }, { "model": "openai/gpt-oss-20b", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792956176-ymZ2fYRACWbL1YaziuQ5", "proof_sha256": "957444b5ab299b1b44d0a8510ba3595bb956e6f9e012694a2a68b0c973c97233" }, { "model": "openai/gpt-oss-20b", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Darkbloom", "run_id": "20261025T200000Z", "generation_id": "gen-1792956196-U9s1IHVBBfpnLw6ucnpk", "proof_sha256": "1d70ad5eb167fa1dee2947551dcad081dfef81c25cd81b35a75f5fd42d4841b8" }, { "model": "openai/gpt-oss-20b", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Darkbloom", "run_id": "20261025T200000Z", "generation_id": "gen-1792956206-qeaFHmLX7r1yRA12j6jX", "proof_sha256": "bbe951dd150b263c6f7e18821c0bcf903dc40c45619754958bc04154a9e81410" }, { "model": "openai/gpt-oss-20b", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792956221-QkXV2O5PLYuKTcVhibFt", "proof_sha256": "d27f0edcfcf6c0b697d29a3d51c6c7b8d9675dc535a18414e8ab5ca02a465a3e" }, { "model": "openai/gpt-oss-20b", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DekaLLM", "run_id": "20261025T200000Z", "generation_id": "gen-1792956242-SWCIqUJRRqSBTEaGBJEy", "proof_sha256": "ccf76d8ad4de3bfa97ef7d3bd24f24928b4e537cd8310550b545e7194695c65e" }, { "model": "openai/gpt-oss-20b", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Darkbloom", "run_id": "20261025T200000Z", "generation_id": "gen-1792956254-rP6ehuFRFq7urHIoj42t", "proof_sha256": "4f2e8769a956a6027419e3d983ee1e0ee8a08b8591b61078498b708c164ba2b1" }, { "model": "openai/gpt-oss-20b", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DekaLLM", "run_id": "20261025T200000Z", "generation_id": "gen-1792956268-a2vws2X70ViPRTx1pzKI", "proof_sha256": "e74af6d8c8c5097dcecf86081230c2cd20cf6f7c79a1a38635950fba20a3375c" }, { "model": "openai/gpt-oss-20b", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792956282-cqHjX0XpUMnOM6XoFJSy", "proof_sha256": "372a787413a0a446cdffa769cee5b9d624681da50264a52a359ad44688d24f44" }, { "model": "openai/gpt-oss-20b", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792956289-CJnu7wI1doz16hRTY0Cq", "proof_sha256": "18929efa45819d5617f1ac07d7b63cf65098e3ae16c3ec4e97ceeb4bca9d604f" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792954184-dMu6uV3BtzvtyWXTZpkv", "proof_sha256": "21a1f5c0c8da88edd412bb0a492968692f0996e05ee52cb0fe341d4d15fde748" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792954256-tV3VmzzXdEyQzW2FjSeR", "proof_sha256": "1ba93b35110da0060aa32237d963296356fac731fb82151f29fa9c361008fd36" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "the correct answer is $\\boxed{b}$", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792954283-jImUki2YwE2FWxRvkl4Q", "proof_sha256": "a360efe7c2a0608fe8d63e5848d9330f27b1be41cd6d67dc7cc41048d6b3628a" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "$$", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792954310-UzhLLOlU5uDjTQBuzKYc", "proof_sha256": "d809e1b4e85ca86a15b12016d05148c0d52505115e044401905eaba3b7a7465a" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792954329-aXu1PMa1zDw9FLVX7heh", "proof_sha256": "8661322bd5309988f7ee3414fad482895df97449c55c73ca4e2e441e1a465675" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792954362-3v4BiYccqsvpeFYMNV2I", "proof_sha256": "005f1934dc50b52bb468453fbb14115d3b93b04eba2e4d37d538ce51780f5adb" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792954406-1gWqhpIRAsNnhvYZjHv5", "proof_sha256": "74f419f48864cec7fa3dd89624b58d34bd55e121f2ef23db1a9a3080cab15e02" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "\\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792954435-EVvVN3gPkpZai9szx3bz", "proof_sha256": "a0813b22fa6d9e5ac3af0312cc5f846b725e1a823eb6d9373e028b8f647615db" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792954460-PiEyuwQZfgerLcGmOlTc", "proof_sha256": "c3464c292ea4d2a22d7bd143f5f18e02d5b9570fa3989dfb5a3453ce6ef60f04" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792954484-dawJec9Zn48TJT7Jnk92", "proof_sha256": "56280338fa5123743d149fa9edca235bb03aab9802189d932a9e8ac063d7436c" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792954502-CJIpHhFYEtmlP8tfQlrJ", "proof_sha256": "9e8c5409496d131004a91f210f6c0b8ff0125d6d1ad18580ddc7aae3a39c688d" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792954515-9QjTn2Xzjs4GSpXRI7qQ", "proof_sha256": "e9b2e2e43f46710e4eb0bbc637d77d8de6ab3a7374cd92707bdfec39b8a97cf1" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792954528-YKM4bQofZVPC4fC83sQG", "proof_sha256": "e07cc605e16cb180b836eefa29a64a917e92bd506140af78b5d225f0893ed6ca" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "$$", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792954545-f4XPTNWsjcgGNLYEEdmX", "proof_sha256": "24401a290d860ea018a96fec72a9c2815a35fae08a9f7d740f91b7c8e1e349ce" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792954573-wAsdPoz6MToBknAStbYn", "proof_sha256": "87ec082a51d40708eeb07ba3fe73bd7161dcfeccc8cc483050b57c435fcad0e5" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "\\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792954604-EUVu186LzaBwbW2zMWEp", "proof_sha256": "d569df623dbd5cfe3b078979ecbf9b039a14388c0160e337321a25450bf9466c" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792954630-3hMpZB1lH1dwwXjzi8n5", "proof_sha256": "82ee609a61da9b81de529e8e9bbe8f6394596eb22af3a79a74fc206fb4ce2453" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "\\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792954644-XJejNb9UfiZrcyyD3HpU", "proof_sha256": "0e1ca62c10ec4f6d7a36e4185ee8da3f90137d4368d395ff381db34343cd8d7c" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792954667-zNCLtvkmYPyt0hIrahGN", "proof_sha256": "70b84c4ac2f1fc57afa96005083717ba62f25f896d33d99e72ac61b992139309" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792954690-ByjsROdZtcRtwu8duMTJ", "proof_sha256": "c7fe6e046b640a96b01e7695928a4879a1c1bfc208ef4193711b03442f77ac36" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792954706-0PfG2zcn9fpyrOVgp4hc", "proof_sha256": "3c57518c6770ba1daf8b584d06fa83ebe4b998dbaa7869aaa4781f27bb46631e" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "\\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792954721-jePiLd7k5yMoTKwtHq1Q", "proof_sha256": "f229056a1b0ec9fa85d8c69d733fd81cdbeccce59b83bc4599a93f6064ba154e" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "\\[ \\boxed{a} \\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792954755-mCLFSZr5NCOYtmbb69oa", "proof_sha256": "7edc93dda7ff390cd740ab253b6c4bdf2aa15f68afae4ece11ad37b360525eb4" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "\\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792954778-8os4GjBseCcYNyqu8CFr", "proof_sha256": "c429f6dccc95cd66ed558bbc44842ff863f7ff6a41d16d8a4c486da45b180a59" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792954806-3qOhjSczkkray2wF0U2g", "proof_sha256": "a280a9fd455256bd1e4641e4aaca9b2d39cbd0a111b09052bff9a9fe776ffe33" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792954830-HTnsjP6WG8baWSebPKEi", "proof_sha256": "2a489a9fd0c551ccb4c451d29c7dd527745d21ab321fc8f3ba63f8e588361392" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "692", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792954852-sARipVNv4c1hXUnXC9Kx", "proof_sha256": "8b40732f5a60b9213ade23e5b7f844f65b36b461d7ef949bc50e38c68183d616" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "\\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792954966-OSnsXxgAhJ99QsEWRPyw", "proof_sha256": "8fd9df041d8943311c2dee9c6167aa89674ad735b886f1c3e185f77d293ad192" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "\\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792954987-p4AZtXI4BNYVQn88lX9U", "proof_sha256": "7865c76f7ca2a4e10f822182121b994a5a2d4d8bb846734924a9fdc8e78012c6" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "thus, the correct answer is $\\boxed{i}$", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955014-SZb0thS6LKWYu86d9J4M", "proof_sha256": "e52ba93ef960c91c26f12ce7c8acdf46efb8478b71e34c2b40ab69eaf94b2c9b" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "\\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955045-AzQ0I8FJ48paWEdp1oYU", "proof_sha256": "8d95268947270c84af0bda713cec63e31571e4b379af99335344f245cb77c311" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "\\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955068-PRqrLnbmmtbPZ0I0Ggvq", "proof_sha256": "b272b36819338d84d3f634d90d983246978d2baea2615875490e86ded5ecb0a6" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "\\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955086-xxHfd8ACcVYO4MnXPTV5", "proof_sha256": "d3b4796d4de521a95d1465dbb15d1ff429cd17b3b2877020217009320181867b" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "\\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955108-pMdtV72XtS8JNYAzoRLp", "proof_sha256": "979d758ac3d3973a0dd66fe19ff065d9573ed1aac7dd001aca065f07fe9d6df7" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955128-MoCvPI7VF4HtIbAHQwMc", "proof_sha256": "15f3a94229333db9ef68f14a1fe2929d0f2199d86f023818f3a5b56ab304dd33" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "the correct answer is option a: \\(0.00 91 1 \\, \\mathrm{m}^{2} / \\mathrm{v-s}\\)", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955155-2rr9cxhKhUZs0lxVNObr", "proof_sha256": "ab94b73eb0856756c054da1c39d108bb58f136b1d3b8883d4d981032db039dde" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955178-vTmhJxzZc6Wa12Fe6Ve1", "proof_sha256": "4db316be1d035d0915458e299a67f5d20c75558c92edb4bfeea9c4ab907e468f" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "d. chronic suppurative otitis media with cholesteatoma and sigmoid sinus thrombosis", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955186-vS5xsngFkxhwqustjp1q", "proof_sha256": "9ca654ee9c7b8001488353cc225b93a5f0bd0136d139ea4d9f8af9eb9997ac0b" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955193-FqXb27cX0x5OEqV84Qo1", "proof_sha256": "9de6953c5f36445589bb349864205d6717079bd33654d7b36b2139c56ce90c32" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955201-s1BZuaJp93ZbwkR8aTtI", "proof_sha256": "1e1352b6a0907d698623d2c3602093a97f61253e675f54364269fa854f5fef5f" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955211-aEFwaaHv6Ph3STmkcEx5", "proof_sha256": "ab2cbefcd2565f57954c1d7df6c834a00d550ec822cd6709c85611d72dce454b" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955218-e8zF5OCxIWRHLSVnCzZT", "proof_sha256": "82271bb7ff4a205ff2b3bd140bccfd9cd90bfc03413765824d70017b91478103" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955225-rFrcwuk3Wcu7fvbvIp44", "proof_sha256": "f9bb230a9806c20508801e5cec59b7856494d9cb5949ec45dc31c6e7ef829ef6" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955233-YwLYSilIzGYQGE7pApQV", "proof_sha256": "f2e6a24bd3147bf85460488834f205eacaa67aeb3ebd5f7f8d19c5de076a31a1" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955257-dNXKtQZPo7yVnLbgo1jN", "proof_sha256": "059055dd89297d4b49c927de97d8a96c1d7f30a2a2f68df0a5c14d2d3cb663da" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955273-4LbZB3HZaPa8eaIiiySR", "proof_sha256": "e6b2a547e5737991d0e6aa1294da6df84a6a33ab7fc8736db495604b3404cba3" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955291-H7pAaHQsbgkZTscw0OPk", "proof_sha256": "90a978b8ecefddd7575bda8677c9b1abcd0cf44162850ed7b32e8123cb6aca65" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955299-xKj0rk2JfJ563T6XFHvP", "proof_sha256": "5ac54e8bedf170a870674e9c40d001d8e70aadc8c8f46079f00f414765ae69da" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "f. since bank b notified company a of the transfer of the claim, company a cannot assert the defense of the statute of limitations against company c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955330-rxfDQKdUP9dwmEAWTniM", "proof_sha256": "197a4b507325f2d6986a14bb2cac1fb961079d13ecdedaf432d8192355120505" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "b. a crime of negligent omission", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955340-KWuKnuPfXi7QAybHhGvT", "proof_sha256": "92a86d7dec4935a344f2e0e11bb4d4fae2cac37f42bf3cb8265867d528c3980d" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "a. no, because it is hearsay, the janitors statement is not a public record", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955347-MAFUC1pvlgecgLyXsz8N", "proof_sha256": "0e9071c5331860ecfd22dde548ce26be09313379ec1341f0379435a82fd2a299" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955386-MZnN1T5r5ZINmYkCLJlf", "proof_sha256": "e5c9de0e949bc6bf33e17c5f3f8e38dd3db8fc057b3f9a3c55786ead9ca6c635" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955424-MiNhq7SERrapTpn5axM6", "proof_sha256": "9747e52bfaefb32e7b959eeebdc565c949682df2828a5c0d2d82036cac3abc87" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "i. prevail, because the doctor did not follow the hospitals rule regarding blood transfusion orders", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955463-5ML5eyw0QGaUHOFTlKvs", "proof_sha256": "5023c497346b3ca539c2d44128309cdd3aa3759eb166384670b9b56d4e5cb3a6" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955501-OZjjZgxbNqhXLNKNzhxp", "proof_sha256": "77a1a0ed71947b8d1b39aadf6faacb86e902b384bfdead82ee159a5539a7736b" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955539-F1GQnVTjTeUWHNCOy9QH", "proof_sha256": "520828bd792ba626022df113b172d683f24cfd2bd69041ed85f76537e4addd88" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955578-p2paQpKqxQVlEF78w7nD", "proof_sha256": "516aad0410761e519d1706d22815185aef19a62483765e2e2a6316fae1355b1e" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955616-4jnvwGkMIaiX8DtzVHLE", "proof_sha256": "0a5a5b84b9994c03c5ecaca7f6596459b051fa005318287843f5255ace6a61d4" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955639-mVrkjlKZBi5bgZoWX960", "proof_sha256": "bacaa610d0ee1455818ea82016cc5d8cd080524132a79e817241bc3efbe5cf67" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "c. psychoactive drugs include all substances that alter brain chemistry, regardless of their impact on mood, perception, or consciousness", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955670-UayZ5EAARkJ1ny818krl", "proof_sha256": "a85c8d972312cce5315f8d80c0436d7272df3cf5a72626b53a4efd4c87d350a1" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c. increased spending during wars, with demobilization bringing spending down but not back to prewar levels", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955709-pOggR3Gx6ppB5We5mruG", "proof_sha256": "8b28483e83d23ac945c8a9679eca54803385f515e9e8fcdc0d254d8eb2b269b5" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g. 5 permanent members with veto power, 10 rotating members with no veto power", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955748-BEqenD8xqIk9WKFm0UpU", "proof_sha256": "bbf5fe77c64e1c6d3c6de2bcfe3d1e7b341c43d3fcf702b44d6685811d9b1476" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c. it indicated that the us would now view all local revolts through a geopolitical lens", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955797-hpjcgbxs5TMm8VARomVY", "proof_sha256": "d4c19d632959998cb127fd2e035ce7fe1fdda12c7326b8cbc1ba3119df2b326a" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955805-cj75bzNwehPShe3w2Bca", "proof_sha256": "cc5bf585f39f844a9b4262219b7107e8936d15e2b68b8613bbbf91ada0512937" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955845-FN61i3MgSUK36v2SOfBN", "proof_sha256": "058ac8867484182f6594c50e1a2b7dbf61d5db51d5a0ca87d58768a854a4ecac" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955856-1Pyvk3j3BvxvngqTNA0J", "proof_sha256": "3b579514ceb7a17246d066425612660b7eb3d104822224cbde6f1f9e5c58793c" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955898-LJkf6eo3g1Gkz1wtkaAO", "proof_sha256": "f0d7a649bcadc54f8265c7f7ca7b74d13acf9d2ed2e5115ab8529afce4df36ee" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e. exercise and weight reduction program", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955936-GOeAsGyNlce2meCk96ap", "proof_sha256": "f4cb4cfd8ddac1f34e6a337b408b3e72b9f26e5b052c1a157463cb4f8c0cf511" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955974-Pf9aJCMTA1gndHWJI5nA", "proof_sha256": "f0539048d5cede740f4c35b6309655f4b7296713515e352f817f3d7e7c0a5201" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d. goods x and y are substitute goods", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792956004-wZkY88iXSGoFtUbtY3Sf", "proof_sha256": "da495066b6052e5feec4aa08315f6c9f4d23f53b7718a0e7bfa11d377c8bfd6c" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792956012-Ej8Qx2d5gOTcGTWPqPWP", "proof_sha256": "29533f3f2386d77a6bca0d86f57dfaac5fd64a7a67ba5f20c567edb6ac2281fd" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "\\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792956019-z0aBuU5zHpgmv14nHM8X", "proof_sha256": "d230a8dc7651008da29a435999f34daf694ecd99184b4f3508c23cff6f328d4f" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "\\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792956052-u2npzZ9Hq7MClFxRxIEU", "proof_sha256": "441b669f7ca94ca95e7cde3255bbe1d982f94c8a01c6bb1c6d8283b4fef69a7d" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792956074-IuXBD2VDJW2ZCvpMJVwh", "proof_sha256": "ac1b8dcb0d26870f833812185c2336be46cc5a687a8aa461f1a64e81e0011b54" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792956090-T4jBY96avqorfjf41uws", "proof_sha256": "92902a62d4f9e90ccee5ae92d418850c4a8336ea38885fa92322852f93ada729" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "when the angle away from the z-axis increases for a 2pz electron, the wave function changes its sign (i.e., it inverts in phase) due to the angular dependence of the wave function. this is a characteristic of the angular part of the wave function for p-orbitals, which are described by spherical harmonics. the electron density, which is proportional to the square of the wave function, remains unchanged in magnitude but its phase changes. the electron density does not spread out uniformly in all directions (f), become more spherically symmetric (g), concentrate on the equatorial plane (h), or increase (i). the wave function does not become constant throughout all angles (d), and there is no indication that it decreases only along the z axis (j). therefore, the best answer is b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792956110-4fsY0qwCHVjqlXigGCRe", "proof_sha256": "b45dfdda4e7f0e8d87a44f8ccd642038e7b2e58f5edf1bcd72e2d96bafd49a90" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792956123-vFJWxxKmIjswj8fNLNEQ", "proof_sha256": "6c78082a3e56164b4ee989eae70d04efb68376013f62e91b8def28ce14afc312" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792956142-C3mAp6zVvyYDnQLkVCdo", "proof_sha256": "1187708fcd862dfc8081f4c4c1bf15f3779465c40579ead40883d61efabc16fd" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g. nutrients and waste materials are transported from the circulatory system to the bodys tissues through the capillaries, where exchange of materials occurs due to diffusion. the capillaries thin wall and extensive branching allows this process", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792956159-QJA4nexcf930SRoeUR8M", "proof_sha256": "8742792e04dcb16c8bf2cef0d2fcd383597979694f957343ec4a838e1d696dcc" }, { "model": "ibm-granite/granite-4.0-h-micro", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "none of the above", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792956169-EyuGQAtm0jJOPNDb93a6", "proof_sha256": "d108677fc5fdd9d56d8c8c21b8f662b931d81b8d9c92bf8f298fac5a0ec92472" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "\\boxed{f}", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954184-Z2hMMAurcOe7ypUkcBYd", "proof_sha256": "9f0ff8e9080a205021448f9e47aaaec81d269fc2a39d7cf6ec43161426c6ef2d" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954216-xk3SdK9cO5DUYEkLf5Zz", "proof_sha256": "6eabc38520a07a743afa8a0f9f3b0896b341eb04ba4128b9c908ef4bdcf6191d" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954256-gu3gESTZGjp4FBGe8xgf", "proof_sha256": "4aea7608801b4074cae798196968a4a7076b2697d57cb75997623c1396327cad" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "lets try to compute the derivative of option c and see if it", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954280-7ZVT65px83GloKYuif6K", "proof_sha256": "eb58ff70a923f4c44f5963c653a0d29af7912207a5c4d5c427a069f7f86d2f32" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "\\[ pn", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954310-131t3HwSeyF8Kt4ZRKnx", "proof_sha256": "315968d281004fb79b1e94ff1f41a5e050bf93b238c66357f13687f65e84f0fb" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "absolute value and half: $a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954338-SeqHqWA54wSHvQhINOnT", "proof_sha256": "9cecf7f326c1c6233338e7719bb50cbbbfe72a3359cc417d5b60eac7faaca56a" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "x = \\frac{kb \\times 0.0", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954366-oDR53eEzTGxhSIXBkDVz", "proof_sha256": "63dc98c76fd2eebd306482ff392a2e20163275817d19ce5bfdedcc03b3b87a2e" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954402-1ZIF4pcahRQ2aK0JCHLm", "proof_sha256": "07648091352f6272d03d7a2ab1585cbb4a02e3c118e003b154484bf1f08ab6c8" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954420-U4HQ9QAazMFPzI2xlAyc", "proof_sha256": "238b51027d5bd32fca9b56894f352dbbad80b802df0b758f9af9f16c715dd0c3" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "given $a = 4^{1/3", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954433-edCPDXWDrpFXE8IEx5xp", "proof_sha256": "357c26e405be50f4b703f91008f45d7ddd497ec24c0edf836df4086e9935ffa6" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "now, express \\( f(2n-2) \\) and \\( f(2n-", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954461-inrj4rUSXUvD89OyffQ9", "proof_sha256": "43ad5852ddb8e256ce68f9a62b6daf286eb3e20ec91621306a65cf46970f4913" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "the residence time \\(\\tau\\)", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954490-Ic9GZmD1iJNXJmbBz1WA", "proof_sha256": "7a12c1af7cee1606967f7acfe395123e9abaeed865f0705f35396c96665190bc" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954518-n6cvI1sGNVdAFv4ymRQu", "proof_sha256": "519310d182e78339ed344bd5257cde68d46ba000f6eeab4d24ef1768c88c0d22" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "\\[", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954529-EqAR8wcZOhkZU9lfknMf", "proof_sha256": "7fb4b7674d28e768f7bc89159e7c3e6e269631ec6f2117b97c3711fad5d5c117" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "common part with previous cases: \\(2^5", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954544-2WaAkUxxxTEW1fUvcHyJ", "proof_sha256": "880b4cee61960f32c3841ada3a71e10ef958efef01b416493fc7667c81f2780f" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "given that, perhaps the correct answer is to calculate the energy based on the speed needed to cover that distance in that time", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954570-H6yt6l3ZLUF3BO3Lw6yA", "proof_sha256": "710f6e328aa604694ea8d0b6dc8aa1b14957f8c63b1953dec763897444d03a3f" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954598-zHUHeDT4ZrH7SuCiRzdx", "proof_sha256": "5dc713e65ce097cd3a4c21520469019accfb23b079c3eee6c21b940e3f87812b" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954621-yC7A41NVC6GsZWf4YuGv", "proof_sha256": "26b43566798b8b949c5f1423739600cb3380ae875d8494c24e098edadb9f7fcf" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "\\boxed{e}", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954638-Oazrzmov6TzGE2v9vrqK", "proof_sha256": "37846c276509249257130d52d1a59d2008c35ff1bef5257eca3b023908526426" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954658-C8grNEwRFapHThSC3RUT", "proof_sha256": "4204bb97401a4d11685d7f2a0693ab56431f48571d188f38f773f7a831c57645" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "let", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954672-nEewk1uIdMEcfNGU5yXN", "proof_sha256": "f1782c040f03bbbfd900814ced25e9011bb010c4dc433733023588896f04dd06" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "this is equivalent to", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954697-qLbFErSLGia1HC9zELKh", "proof_sha256": "079204dd69d9b9608bef7ea012ffc3d3c139b42c070b2625bc91afba037c5dd3" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "answer: j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954724-vCXIVYluuVW3ThG4w58i", "proof_sha256": "8c5df246cd8f1464142d224e3d528ff3f9d3a255f41a7150cee8aacac709cc94" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954734-JGCPZId8eqD6UpwkT0UE", "proof_sha256": "e12711da21bef2bf925b9675f87dac790dcce935680d97ed00361e16c9700d88" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954748-jr5hSbMYsnkIhpiZzMCt", "proof_sha256": "b133b8116e7f16f7bb4542c505f03490e32067d1e36891b448fc29385444692b" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954759-mlL3UcelOOm1zDFquEsF", "proof_sha256": "bd72efa624eae294083f89e1e371b00d6e6fd4ea48bd910713464e70663e96c5" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954772-8Yerr275Odum1i8eybST", "proof_sha256": "1fad81f92754c40e995cf4f1b10ac2a0404bac0cf6aed71b7b0ed01e823c5f6f" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954796-HsYgHw3Tb8JU3oqqKQPG", "proof_sha256": "151fe3bda7d9958cc720aac7b10f82156889f36e14757654b656c9f0edfafa7e" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954816-dhPSMzZswiY5Ls0y2ig1", "proof_sha256": "b0b3b3f524a30e2deb270cb6c24af6aafcdaad9bd25504e9001b40d2f591f7ee" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954844-rDJULJdd7LpkVqVHraGp", "proof_sha256": "6d8ed65254471eb4e5776c95e16a48434439939af9ea760cf0a4683865ae9c80" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "with \\(\\mu = \\rho \\nu =", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954870-SRlYAz4hujiKrKz6Py6f", "proof_sha256": "20e3c80fa56ed429f2c9804a5d29909094cb2044f64026287989a4edf7f46a8e" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954893-hUZjthN3qkFEDJ41fuLL", "proof_sha256": "5731656bb7b179087479412aebe959de0cfc3931d8c553214e182e8b79aa9297" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "given the confusion, perhaps the correct approach is to realize that the activity function is the current activity per fission, and the total is the number of fissions times that, but the activity function is for the current time after shutdown, meaning that the fission products have been decaying since fission, and we", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954907-AL2jAxK8tckvwtxEkJUd", "proof_sha256": "0dfe3f5d546563e641ca10ade4dcd003246ffc6740491cd1e8e8aac0416c5122" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954933-W0c4UmTLCvqYb9U69ALO", "proof_sha256": "bf0cdcfd9d9cbc92fa966abc24e178a857c929334b1d5f7271bcfcc50c8e6686" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954943-Eto4eHz8L9wuIQz7jDuZ", "proof_sha256": "59b405cb4b66752e1d8f3a532ff6c122873307eb727ec9c458e81a2425e892ad" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "we can relate \\( vd \\) and \\( e \\) via", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954962-8tyyfF81vsbkzMYAd777", "proof_sha256": "bd70f4e8a908ae75b0f9974320c55c58b637d46076bfec8c84fcdb7ea8fd3981" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954986-xz1uCWGKJ6calH5hgJUp", "proof_sha256": "ef8b94f0725ada3d5a1772d2d19b95e239ea6f2ca7d840ddf9064fd59a9a4297" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954994-98UsgGn7NIalWXLyHnWM", "proof_sha256": "4b2a9751dbf1b2a6805d41d6c4397fe4747f4081582c4db4761cfd4770912317" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955003-gntep8g2Y52lK7FyNgmf", "proof_sha256": "35446712a22bab5b5cad15f6cd073f44b96ee67278baa2c5b3486ad618676ab4" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955010-lePWBhgCluQDCjNUPQMm", "proof_sha256": "6e6dba7a1ed506835f67cd1b58c1356671d3ef722f265d333f7f763921c34dfd" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955020-8EKr837oMTryq3YPrr2w", "proof_sha256": "d02e1474913c293438098fb0c4bf9802ce8cbfac76bbf000b2926f312a07e6bb" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955027-AbTy5kLR0RHhF2JaQbDV", "proof_sha256": "5f71c712c75d148fa75327e46aebbd94beb81547fa912f27ec8694770e4b0b0e" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955035-B0MLJmQ837jeAkheSJ3Z", "proof_sha256": "48d013908f17067719129e1c8a811e2cfad54589436afcf297acc5e36e073a26" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "the molecular weight = mass / moles = (3.4 × 10⁻", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955043-Sg0U3VhhJ42Jnbt9d1EA", "proof_sha256": "245538e79934814c25d3ede99878a79d2ee866fab1693ba63cc29e1c73f01b1a" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955077-fhCuiRzFJfBI7oo6dvRY", "proof_sha256": "4870c20780a8fc4c9ac6aea19cd1b758c0dd6f60ef700be2b2aca77aee46954e" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "however", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955096-eVaUeSiwsYdah8kjrzzL", "proof_sha256": "c12fefde03d73a6308a110222ef4c1c9e07801fbff55e365c57954e203224e27" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955123-yLuXNHtx1UdX89a2NFq8", "proof_sha256": "2344bea404e467245f909276fd1368d0d80a907eb49268807b1825c6e1a2fb00" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "since \\( g =", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955130-qkc5RLxSZBakkrB5tMwc", "proof_sha256": "22cf42ee5876ddcb89b8da60e39a63b0d84c9bd3b0b6955a72b51edd51724514" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955155-WYaqtWr0VtkBMdRM0cR8", "proof_sha256": "99df08d5f2364b90845952abf2e0d01ee7526936970eb4b8e2c17776f516526d" }, { "model": "mistralai/ministral-3b-2512", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955165-06apJ2CjGRzKli4eMUTu", "proof_sha256": "8627d813d86e4257c76d6e19953aa24551cd544dd6562f49fd6e019bfe10c322" }, { "model": "mistralai/ministral-3b-2512", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955177-KIk8E8zWsJS4Wli25SpO", "proof_sha256": "d7db6ec414e5ec58bc6ce1345455bff1c3e92870675c20166c27f97f12815ccf" }, { "model": "mistralai/ministral-3b-2512", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "answer: d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955184-8bL7xP80hXydvGnuysiI", "proof_sha256": "73f5baf4f5fd7571c115174701a24a0e82ffdc425028727c78c11e7e7f313bac" }, { "model": "mistralai/ministral-3b-2512", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955194-s0pGBzFMDTv8MCXRnY6f", "proof_sha256": "8bed8dcf9b2b431948ac9499a327904e427cd51ad8a7662d7f5d4995f82fb509" }, { "model": "mistralai/ministral-3b-2512", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "- j is true but less precise than e because it focuses on ensuring compliance rather than the direct breach of duty. e is the most accurate legal reasoning", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955202-A2Y17CSYzq8R0K7lzgrb", "proof_sha256": "567ebde309957614e94c36898740b45ad0b5ae13b1819157202e79a05e80f9c2" }, { "model": "mistralai/ministral-3b-2512", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955211-P1G7QfviTsC09ZDWZaQl", "proof_sha256": "bf0cfcd455c67bcff901c03db8f317506a4f69d13fd67a390371ede0b76c4296" }, { "model": "mistralai/ministral-3b-2512", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955219-GDQ6wiMPwYIn6sAdPXhg", "proof_sha256": "831334d24ab48670acb6e6dcc1a01e64c3fce8d90a7079f28b0d0b00c9781920" }, { "model": "mistralai/ministral-3b-2512", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955226-MU4fupsQFEnEdtRL1G5C", "proof_sha256": "4d21dd9ad046cd733dc6229d84c3475434f7010ac647f521a64fd7b68f521752" }, { "model": "mistralai/ministral-3b-2512", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955233-kWFx7ECimawqSCZceiQR", "proof_sha256": "08b50733213a0641dbd106c8666c27b9536a04d2102f0f71daeec2c4b05fdadf" }, { "model": "mistralai/ministral-3b-2512", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955241-p0FbJQcEAfMzkJiOaIVS", "proof_sha256": "cf937ab19c3c0ee7061c403d68cc3540d4336cd6b868ba36c611cec5e7f6bf9e" }, { "model": "mistralai/ministral-3b-2512", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955249-wDbImsHf8lngwQyOUVAQ", "proof_sha256": "3ac4d4387cc8ce8d99334a1e2906a44bdb1157ae0b2c7f96056956aaad11a9f6" }, { "model": "mistralai/ministral-3b-2512", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955256-yReOHEWJDGHF0vcKAJTX", "proof_sha256": "d813a9dfefb5109eb10dee15854da0476cdd1fa18a1d58c0aabe27a5f77d81dd" }, { "model": "mistralai/ministral-3b-2512", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955263-HpeufRkcHgh95r9EJJtV", "proof_sha256": "c89cb372ef356a87f492b63e7c60add1d487bf7fae6dc945db1e430bf91b4f8a" }, { "model": "mistralai/ministral-3b-2512", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955271-NTGcRduBCRaEuELmEyL1", "proof_sha256": "509b264ec80d914ee4f90b24efafffdd8c947b5be2dbe282acd99ba0df8085ef" }, { "model": "mistralai/ministral-3b-2512", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j (with the understanding that (b) and (c) are owners equity, and (e) is an asset.)", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955287-xPPimL6KBVA80qhJRO8g", "proof_sha256": "4be0833e8439b50f8a62d7c12d2d1802c78f60a519c751d556a0b96501dd07ba" }, { "model": "mistralai/ministral-3b-2512", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "given the confusion, lets consider the question might be asking for the tax on the par value of the stock, but the tax rate is applied to the par value, and the tax is $0.0", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955296-1NtjDyaESzzWnVkZJYVV", "proof_sha256": "b17eb65afaee188ed25e62666bb160602c20abfba6934994da995f03773294d8" }, { "model": "mistralai/ministral-3b-2512", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955319-gWP7opKocM6AXALpJ2T8", "proof_sha256": "902f5a75fa45e03c45982ecb45247b8336e1f257731f5b136c5d39e622ff9b61" }, { "model": "mistralai/ministral-3b-2512", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955358-twADS5Ry4aMkh94n0dOM", "proof_sha256": "80201a4c8e390f77d09f350737c3188abe385ebfba7f3db1c27cfb4248f178fd" }, { "model": "mistralai/ministral-3b-2512", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955396-3DmWpU7P7KunbEMlI3jJ", "proof_sha256": "cbc3853faeb27adfd70d0f0ba3115725bd953933603471fc1e774ee4a3c84f66" }, { "model": "mistralai/ministral-3b-2512", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955434-Sm5xy85RGNj1KOAb7CBZ", "proof_sha256": "5b3356320e09cd4d6638d92dcc857c941b33b777acd23094fd7caf3cf21c6896" }, { "model": "mistralai/ministral-3b-2512", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955472-MtpujT3nDq9tWrGAIPJy", "proof_sha256": "239502da9901090c746d7ab878b18472783ceff7335f2bf8aa6a44392c451f77" }, { "model": "mistralai/ministral-3b-2512", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955512-xzmDNhKWW6pYHWCu6ABk", "proof_sha256": "77f02361c75885fc366082563ac1c4a4a6d2ab9ea9ae6aedd366502ff47321c9" }, { "model": "mistralai/ministral-3b-2512", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "- total term: \\( (16", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955535-p726dovUHFPdzbh9InqB", "proof_sha256": "0df097d9742758a8e1a5942dadfbbdaff8633dae165c909bd552cd954f779503" }, { "model": "mistralai/ministral-3b-2512", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955585-a8zF9snIhwZjfg9ltFbA", "proof_sha256": "0460be8395c70d697544cba037a6997b1fdc4f282d762effb93e01a7e9721858" }, { "model": "mistralai/ministral-3b-2512", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "h (2.2 × 10^4 ft/sec)", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955626-Q7KuQJcx7CVvINbwelaB", "proof_sha256": "c114cae572c037824031f018557c9deb444795177f7dc70f6cfed5edc39c5f3f" }, { "model": "mistralai/ministral-3b-2512", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "} 0.94996 \\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955653-A09jGL9j2xvdDSKwTDKC", "proof_sha256": "45068862bda9b6a37e8ab54924dbddf913989d2819d45bf1a68729a6125a7f9e" }, { "model": "mistralai/ministral-3b-2512", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955676-ATYSqXvRV7aspnCiKLUA", "proof_sha256": "f9f82a7a95566bd158d669563d4fa0c05840b7c4f1e2886907e4a90f4133afd6" }, { "model": "mistralai/ministral-3b-2512", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955716-X4zNpOFEB4d4Ae9pTc3c", "proof_sha256": "883c8d5e5c0711453213be766573968d3b26eeaecb7d26904d4085769cc9b325" }, { "model": "mistralai/ministral-3b-2512", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955724-mnA0eB7bJ3dYNooGJshB", "proof_sha256": "d98caeee4a466725f6daf427f7262cc68e941f297e443eca4db6d2bd3e140900" }, { "model": "mistralai/ministral-3b-2512", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955767-quSd3F6kouVjslbj4aDy", "proof_sha256": "6ec8f766e545a1ff465cf65d8e88a2ad1c7e90e7141c6e504e1abcc20ce7f9ca" }, { "model": "mistralai/ministral-3b-2512", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "### using the formula from microwave engineering by", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955797-zCxtS0F8TxhSNXrv1kN9", "proof_sha256": "770bd12d5d267f0b1eca5314704b3e2763f26c4debfa9ea1f9a60fb19e0cc56a" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "thus, the area of \\(\\triangle abc\\) is \\(\\boxed{256}\\). the correct option is \\(\\boxed{b}\\)", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954184-M3YYIgFUdsG3qGbUIPH7", "proof_sha256": "15be7b8ca10a0c1f6a154447c294fc00c93de01da54cbc3fff0e6a84b6c05ff5" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "\\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954193-HUkdugNLUXONG4L5t0OK", "proof_sha256": "f0903926885d8248e5b37df5f59662571ad362e068dea0568f09de11c41f822f" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "\\[ \\boxed{b} \\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954256-pbKHphhoqjfPaXg9745W", "proof_sha256": "53c0de915af4d1f93b86b29a33d32f319e9a739def4a6ab4670104573d8c201f" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "\\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954265-qwvNivIXDC3Hfi39H7f5", "proof_sha256": "cd80ca61718952f5bc41a452bc2bf1d6b72dcb3b77eae5d14bbc9c4dfc1b4fa6" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "\\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954275-sAmMBD75lKLWXrpLYs2Y", "proof_sha256": "03eb24b393c237538013f5fa63d85fd9f8dbcd36cebe7860091a56a262209a15" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "\\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954285-7me2Y5QpgnmWLRGxwph3", "proof_sha256": "80c893d3b66bd7eccc3c3f243b5e13c2192c5d65b139b12dc6c64b6b2a61cd99" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "\\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954295-VKCaLxbtH959IQmRuDeM", "proof_sha256": "78a9db64d462dd2d7fd7f3ed52880c56b2e20e0870e04bbb19784fb5bb578621" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954306-lMta1DfW2J11lUk7NB8h", "proof_sha256": "9c19c6b091e82a201f0507344108f45d3ed99ba5d48ce5e2ede069f67697199f" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "\\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954315-l4WZqjmWRfQTpholYMlV", "proof_sha256": "77586814aee48368d205a396824e63db5f5a7eb45f066f68720031f42753cb79" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "\\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954324-3JLEReIuU2ImLLr7PJ7y", "proof_sha256": "bc3739e8e9f2ac6a94b7fc882eba64732024ed0e26e10465715647584165fd74" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "\\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954335-WK1PuoesKlSy7inREKU0", "proof_sha256": "c5271afd483f3be49252ca9dcab382504c260fc152a58cbcf171a091309f567c" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954349-nrNHaF3srQryzfUrXRzN", "proof_sha256": "1273af35515664a2cde142b3504d6cbe73e045546816a02eb7327da83fb6309b" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "} \\boxed{e} \\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954358-TAQmqFzoZcRoP2rwuPSk", "proof_sha256": "cfe79e4055b5926596a7454cfc4c2a967d7e18d1eaa23952dd29fe3bbf7382d9" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "\\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954367-9OKsafAZNgu3Q0Vxt0ZC", "proof_sha256": "5ab753ee67467b858bf365620e59c3f2b1d7ce9c4ec77d1b14943e5af785f1d0" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "\\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954377-3AkuApdCZpS3geABFvWN", "proof_sha256": "d2fbd33a5eb620b2862461b480016539ca39c01dee455652bb79b40b40c4357c" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954388-ETonpaSvudinQpkXgMBT", "proof_sha256": "f20630f756140a2e8cccd916534daa9074f7934537170d705fc760204d1043a9" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "\\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954398-nofmbmDlGEEmSLLFxQdX", "proof_sha256": "8b2a5e89aff4ff9a836d4400d72ab68348263ebf229e6460bace0184570c0b53" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "\\[ \\boxed{b} \\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954406-OsXGVZTIcPCCp44r67tB", "proof_sha256": "428873018887cc546b1433415389ba9c9a893cdc8149bf170780131ef88ee658" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954417-UkFrglClxvGEXSwINrzf", "proof_sha256": "0540c770abcda79c5a4a38220fef89307ff24c9092d22166a12e18d658f4b397" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "\\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954428-l077BZiUSZKzmiUx0J8N", "proof_sha256": "9f8425b20ed6977f469208bcdf88730fcacf03ae3e665b51fdff667208232a12" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "\\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954438-AddiRJw3s96BXXNrQgT6", "proof_sha256": "39e5cf65cc5af15f9cecffc351566df75b444356f862164bce21faaddbb73e0b" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "\\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954446-48CH3ibCT1NA1q4iNE89", "proof_sha256": "13d989c23f2c0afb9e29286d5202294fb7b688c0761ca33fe1ae77306dfb2fd0" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "\\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954458-mWJn5uXSTongNLiHOInh", "proof_sha256": "b6d5f2c010a693ecee975aa69efb1b3466d07a7637573c0358cc8586065c619f" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "\\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954467-HiQvBA6SCO5hkUYj01He", "proof_sha256": "64a0b8172f68e118abc54095c328dfd2c2dd1d5db0f314eff70c7deadb12058b" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "\\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954476-13Rit0wr80gHOonNd7G8", "proof_sha256": "903baebe8b575adcd21ced512aa9f10c01b2769be46e7d1f4983f5a17b280e94" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "\\boxed{d}", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954488-b7rNNiVKIBqGg8lHumCa", "proof_sha256": "b9408ff80d76fd4c99fed28f9241b5bd33ce9bc56ec3080f6f4bc1559dd85c43" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "\\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954496-ImWTbO0XHJPokpQWiVaG", "proof_sha256": "6e62d9b89e359a76fd60e189cb201e883c72e94662775c955f72f8ac518a81f8" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954506-AKP9gtZiGbMbWjVMeY7T", "proof_sha256": "b18333deec35f6cdaacfd14a14577525af373e27c48c00c5a380a570390a91d3" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954518-4xqzZoqE4SSZijddtH6U", "proof_sha256": "d5d4c95e4ab159672c9d15e746ac298fe530b4b104388c62d62365bad3765922" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "\\[ \\boxed{d} \\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954528-oN4F14mvhO8VNZVWu7oS", "proof_sha256": "288e3b3b5365d4f77d3d731dc488742f554a48a21cf5cc54a498307cf20165e1" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "\\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954539-qrioyKLUELjKB2zxiEOS", "proof_sha256": "9d28437f4f0aff212c7a1ac5dc8bbe994b3e6a635b6cfe504bd4f723456742c4" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954548-hXbqg4cNSjAkH0jQf5zY", "proof_sha256": "49cbb6dabe2ae0157aa03de8ed6f393c5481af53867cc0a97d41368431a35fa4" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954558-8aKWBuSMj9pZX3QaMHl1", "proof_sha256": "adc9782c88ffbd222843bac3ddb16401be73cd0ddbcaf441a9df4714a70dcbac" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "\\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954569-CrGkgmaHgpIOTDzARqWG", "proof_sha256": "4bed23311dfc080ff56f4ef97544c32d54920a20b818a2e7207a1597c92b0e62" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "\\[ \\boxed{e} \\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954578-zTSowYtMZBt2gDbyp2RA", "proof_sha256": "6407b1ab077b24b5ce42a5558a1af4bdafbe457956d8e9cc9f717a2dab9f7110" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954589-pXyhks7llbmLvVVkUAEw", "proof_sha256": "47b2b7689647bb49a7989e34ec7711f932d27627f447162211c4710ec40c5c0b" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954598-PgpRMfNd5J7XlEK3w5kZ", "proof_sha256": "c766494de5faaea5366ace16ee78c336d8deb232c5dae0fdd09d0f9ea15441b9" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954607-1JUoOWQrkqSohtMAd1r7", "proof_sha256": "d1affb8090fa286481e3b83e7714edef9a8f2f46a49fe6642d0daea662c4d4c5" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954614-Ch0R0GYfTaxS4iBbKQZX", "proof_sha256": "fd87e2772492ae8fa04f98561c09282b5982702f77660de8e936eb0a5c141dee" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954622-rU1r33J9kZzg65V5EDzi", "proof_sha256": "1634b53b1582e686ea36d3db8a8d95ebcc41535ba903d36f4d647544a26bf33b" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954630-6xamsiiKCt5HE3SAPR9a", "proof_sha256": "fc197b0a92e2ca6cec2f9bb62ad9af41b437f4c69fabd09fcafa77885eac6195" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954638-SgFRH7hP8erIGz84b2a3", "proof_sha256": "ad30c9b8f7e980a0312ca77dca580c7a84780671febcef749d9724bae6c07e45" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954646-gSKpscFPv8b6Igbe8qgS", "proof_sha256": "9f8e3290ef6353b527cac0b08e63becbd79d624c62e74318f558be48e52917fd" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954654-3wXEAoM49cJiSwowYwaq", "proof_sha256": "cdf6f0f661a1b27668cdb2c53f7e0a9b92c0523f36ca1035f12aea51dddff3a1" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954662-WXozbiP6lUUZjT7LTkuS", "proof_sha256": "8ee67ead50062c47cd87a83d43dfa315085131d96d9b9c6ca43b8861e51869f0" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "\\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954671-VyVFEuN8oaXRl0ssWUeJ", "proof_sha256": "52b6e3c0bdb7e9c9bf2eac686a83ab94f2d07c8d3904ef5a698bc67c57cbb201" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954681-4VxAcfMopWzSiF86XT43", "proof_sha256": "c1f3000483a22c79418680f7d857ee72c1d13c0e8f43e451743ac74249fd82c0" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "\\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954689-HXsxUSnux6l773ibhXPl", "proof_sha256": "c16f7b309aed2ee2a6505d7a94477b452d6ad2af2090bbfc940a9d41dcaf633c" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954701-kNvhLZkPylCwCq4CCDGJ", "proof_sha256": "f0a57d0542ae14c3bb1510c91d94f3d7ea37c2ee64d5ccd1a8fc966f0e98f399" }, { "model": "amazon/nova-micro-v1", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954710-0oXOLGhXsf6Qo6BOCRcI", "proof_sha256": "2cbf4aa67f3025cc939501020685fbdf4e155f793424544a209eab2b5135403a" }, { "model": "amazon/nova-micro-v1", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954718-EGu1QJ3ncXSUXpesuGxs", "proof_sha256": "0625783c6d3780fc96b15d32596ab6eba38a3e038074648cc4334220594b8fbd" }, { "model": "amazon/nova-micro-v1", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954725-bpaS4bL2M9tHVWjhO40O", "proof_sha256": "6a9581e3bd7894c1e90523c082aa29f617e776af87a49c2acb10905dff31a056" }, { "model": "amazon/nova-micro-v1", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954732-mdt9GeIX1RSdAAaIipCf", "proof_sha256": "063961c1cca35e98c6478d6a32c9a14d761cdcbdf5c75f11868110ce040e38e5" }, { "model": "amazon/nova-micro-v1", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954739-yBp0vS8tudMFwVVtPJJB", "proof_sha256": "b3e9fbfcd886a195a54d0d056a533ef5135674529ccd065060930bc6f76f9049" }, { "model": "amazon/nova-micro-v1", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954747-o9bKQHgYZDcmI2NNL4i1", "proof_sha256": "6ee0951b28919e9c20c89b1b33e67ce9ebb1709cfdb08c21aa6d46ce32e35783" }, { "model": "amazon/nova-micro-v1", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954755-H7o4trYjfgCIUMtSWPWe", "proof_sha256": "001d0daafd53ad7097119560b704beae90773cb81ac8e4a79e8ee7e456af0d25" }, { "model": "amazon/nova-micro-v1", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954762-bkachqWcve4t1o5MmGau", "proof_sha256": "bb93d0afe8290b38351724c58c33dc0469683771c7b864f05c37a8bfb2b3c4cc" }, { "model": "amazon/nova-micro-v1", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954770-E3JDjgGdKa0WLuYie0aO", "proof_sha256": "e8e064aa1edeb14ff421d14dab5a0f0c0837ad6d816b3d7cfd4a6eead312b78e" }, { "model": "amazon/nova-micro-v1", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954777-9IXIzmR7wTkNHrShWKHt", "proof_sha256": "12cc162a404a9a7dfa8bf1e862e5f1d4ee56cb6fb39d3b85df3835d12240a8c8" }, { "model": "amazon/nova-micro-v1", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954785-fQ2cmGB5NcxXASEaSBQU", "proof_sha256": "ebee8517d152a90e76d1b0fb38402323617e3153d1b901f44e3daddb7fa31512" }, { "model": "amazon/nova-micro-v1", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954792-IbtLSHIWAnPknzX8WCf3", "proof_sha256": "89534aa80f8994b7df3b5bc63425936ac138084b3e8971d220b7f50febe32fa4" }, { "model": "amazon/nova-micro-v1", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954799-JyIhfrWCGWeuoAwCFktC", "proof_sha256": "d9fc6bf407f84b684720ffc3505b0c38f5a4c99943f0e4dd9b4388ead28a1954" }, { "model": "amazon/nova-micro-v1", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954807-zNTqCZJygt6PbT4XqNaQ", "proof_sha256": "c236e100f2cc38f50e830435fa46ad6833abf9f9231f74513f1eca63b8693f42" }, { "model": "amazon/nova-micro-v1", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954814-WaWosIMTW7WnND3QDbLt", "proof_sha256": "a2483db77ac633b2c50d078fd05ad0c4b36e4c21f86453d8cfa09832322a8845" }, { "model": "amazon/nova-micro-v1", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954831-r4s2WrChI06eq6ZCgBKg", "proof_sha256": "3d43f86775ca1a50f95aa70cca36ff6e2b1b237c9a5835168a422f09cadd105f" }, { "model": "amazon/nova-micro-v1", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "} j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954840-bCJR6jN7bEHYBOMpmytA", "proof_sha256": "bf6ab88c3873a5d377bc9afc2e8f4532b9bd5f76e2cd9481b59df2638696cd99" }, { "model": "amazon/nova-micro-v1", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954848-F5o39gMvFIGHu8iI6Uw0", "proof_sha256": "7fe3978dba9a440b339f5ee8e17f29f02341a02b5532e95a8f95be8381a5a053" }, { "model": "amazon/nova-micro-v1", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954855-DBhXe0Id7uzBEMFVwI7O", "proof_sha256": "7b7534d1d3c7d1cfc183f7be398bd578e8353193bfac408cbc7020dba1d5ecb9" }, { "model": "amazon/nova-micro-v1", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954863-DyZcM6hStHr1oc6lFN4y", "proof_sha256": "be3e68edae8e53df10bcdbddeac5e26c9d368b4414596ffef83e47b8e4eb1326" }, { "model": "amazon/nova-micro-v1", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954870-Biug3rfbfjrMjRiQOI0Q", "proof_sha256": "c47e47bfab9eacd938ef0dd7f9fa59106eaad4784daf2e54e6ff2566e9746dd5" }, { "model": "amazon/nova-micro-v1", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954878-HLhL9vLj1Pz95cdroz24", "proof_sha256": "0673c06c56b4365a2cca29ed615648ed6f2206e437ddd3e7ae86e192293310d2" }, { "model": "amazon/nova-micro-v1", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "so the best option is \\(\\boxed{b}\\)", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954885-V8cWg3t640WWVEfhDZOt", "proof_sha256": "addd94899123487c60d2ce76e41938ac11ae253fe460f22f2e38fe7b2c987367" }, { "model": "amazon/nova-micro-v1", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "\\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954895-yY2ph32fozYMIplKyusn", "proof_sha256": "85bd59bd4fb8560929a6386e3dce7802f1872360a8b3e8ebc16e48597f97f4b7" }, { "model": "amazon/nova-micro-v1", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954904-HPv5jFUqHpswjdJwmpgx", "proof_sha256": "7797a5210ced6027890a15e7e476f57a68fc58fe3bdb4e1a06bdc584c66bde04" }, { "model": "amazon/nova-micro-v1", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954912-rBs2NwXX21TNUe29aByb", "proof_sha256": "4d95b761fb3c30fb8e9afd168ff16720e8a801f2a2b8de45b78722168c60ee47" }, { "model": "amazon/nova-micro-v1", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954922-OeOhCrV4DuCcXFR0oPcj", "proof_sha256": "96e9a26a410632d81dba9d7a2660362ad9a29dbda4fbbd0384203c7db8d9340c" }, { "model": "amazon/nova-micro-v1", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954930-0YxuFp7Jsr9WRKlHK1Nn", "proof_sha256": "477aa572ba597210284e9b13b5bcf5506eae9640f1b2bd312039ef1dcf3e9e15" }, { "model": "amazon/nova-micro-v1", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "} \\boxed{e} \\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954940-XU4XwnXSY4BMA0J9g1YN", "proof_sha256": "a43f11f4fa42cf39ca82cb7d00bd462d16b19194b5b4e36845db2eac26096d33" }, { "model": "amazon/nova-micro-v1", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954948-MAtQbjujvfdtnZisgLxE", "proof_sha256": "ee2dca7f1554af038e2038c7861ab1be44ef8835ff1b742f12e7b45179d07342" }, { "model": "amazon/nova-micro-v1", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Amazon Bedrock", "run_id": "20261025T200000Z", "generation_id": "gen-1792954956-p1h7evkyr4Vojb8Qn3Sw", "proof_sha256": "028b272dfbc1347074efd18c643b9935d762846342fa57e8a10321f7be974a52" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954184-gihfR04zZLTKlap0aS5r", "proof_sha256": "1c84c7aadce8fa9d5898de756cf742cd1d8af1668e54626c5ab99f206141ce90" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954198-JPOKpd7xsFWa0RM2dT2H", "proof_sha256": "772cb46b30315cecca0c7926a0f6f1c6253d6c69f723f59a6b4c9f57a4ea09cf" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954256-ImcEfFL0F2DsvZXYNvjs", "proof_sha256": "305ffdb60c9b86517585ef66637c529e6d0280c913b3ffbe11b572db57781f57" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954267-XtYctgR9SRAIPp1Y7jPc", "proof_sha256": "4edfb74aa62ea0753b85f7a3c0aad21f210c41e87478dfa1cbb719bd81b7fcf7" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954278-Rv2mc9UG408QxtmqCVXB", "proof_sha256": "eccf086cfca5d62728ff17b160755ac8ef4d14eebc093394d7106c1f683384c3" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954293-5X64ExibEZgdzWIYhjIV", "proof_sha256": "90248ba5c4b7c3c0a7b92ab06a14281fa62f9be1fd25e9f106ac327df87f9553" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954303-j4rVDU0yAFRuqFurrLdH", "proof_sha256": "9676703fd68302ad8b11d92a0c1b71a29565b1db50d845e6f9fde0e9223ad96f" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954314-z1XP8aamV2RxzaMCKgoT", "proof_sha256": "da5496082697836c2fd44943e7863febf011c13cc8af51763aa18f077a1635fa" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954323-6VF27yDWy28jNUXyIITx", "proof_sha256": "bc8b59b21a260ca71c5aabffb42cc140d13a3849db8202420bfc2556e225faad" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954333-DQaznijqWkEQfJNLLCNt", "proof_sha256": "1ef8d77179ed37a97d494b35dfe07c77157d60130bc3a9c68a85838fe65930fe" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954342-AfxEn6B3Hk1broCq3LqM", "proof_sha256": "82082e0640c12ccb11352bcaed6b2a08f0ad2303899d2241486699dabfc90379" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954352-yi4hUUNSnO5st9NXvADN", "proof_sha256": "6f244a7b61a716a981453ac7f6a33badeb99ef8286638456ccad9df1f049c9cf" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954363-frhLR4GnZ97SkMf1Ufdw", "proof_sha256": "7fcd9ddc6f5290ca6cd75abe7a7fb1634a0af4bde5354af2075c7ffb0c67bf22" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954371-taRmWk62zKy9e24ErWQL", "proof_sha256": "8c772e65ced2db090c7400e27271ba4caa885ae064d81baec795df07412933f9" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954384-yGKMmsLmNZ5l4RkeVdNR", "proof_sha256": "0ded5917711038e906598d3b103e35cda384c7f918df4dac395146b19bc3e6a1" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954402-t7pXzBhmcuN7G8joiQJz", "proof_sha256": "ed8eaa2434a869a6892513574f17224992deecc4c4df977149a3e0206fe0bea6" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954412-x51gZYTIfH6e6xtBfR1n", "proof_sha256": "05363c3faf5990c00e21b48c446e0905c0bd168b2ef006ea3c8acbcdba5a3904" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954421-HF8iGVtvij8m8JMlkwKo", "proof_sha256": "8643c2f72edb0c0f70d2a3cb5b3930198caf7401c0de6e1e8dbe6dd17eebe9dd" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954434-EfrxDKuIzrWXam889fyd", "proof_sha256": "940bf701100eac7444029d03c3b83025a5bd329b088edaa46701d2f70555560d" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954444-TjrCPVQ74JLHUH9iF42V", "proof_sha256": "fcaffd9953edbd007a8c6f3ca63905984fa7490776b8cf76c171e9fcec370b3f" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954455-xiUoNY4PU2184mtXsEhE", "proof_sha256": "648d3d0a27a11a31326665da064b064067f7d626678038259542d37f2fefe342" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954463-T8JV3D0z11bIxVeIcE0T", "proof_sha256": "9d25f1706c1c2a88693f35ef1489880aa3f54fc5a194baba67fd53c1dfba5085" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954476-2ViiXtuvaI21DpR4O4be", "proof_sha256": "231ce732b71042c6277c7f5641bd94d0542c6a9d8d80543cd75d632e89e6e7da" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954486-86lxlfs38keEuVyPwodc", "proof_sha256": "bb9ef61ab9f795d71218e6b9a95dcb9db99782a8d52ec4b699d48eda778e97b7" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954498-x6rs9pEJL7wHLxMipk9L", "proof_sha256": "638cb0366b154e5fac94671e7733052488396872f3e72780dab005a7389dec50" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954507-VkKNkoEw1exXa2Lm0yXy", "proof_sha256": "4f1e6572529bb4819f6656fa78dd5ed9de18038002f8c223c94a11d30b8233a5" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954519-ET1FbGQlK5U54n4JqP2Z", "proof_sha256": "fb85844d5d3013c86b483b9db268fb7666dd4b03aee4e85a551fa2246361a725" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954532-LWVFaz38gmGRbCBzYpUd", "proof_sha256": "d8b884a6ea114aa00cdce05cf1bcdf879e39de8a5b81504cd6de8a4057aca27f" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954547-qNnOX4y2K8OTd1srzQ0W", "proof_sha256": "afe733e2eee0ab4fb6044717f3b238f2165e2dd5d6d3b1a1559523fb1ab5ef0b" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954558-cR4LN0O4fHgrzhvW0hpP", "proof_sha256": "e6c657da2988d84fe718739d5f452c8a7e1518bea914c229ff5bcf3e5165b16c" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954569-eCYksVyLvWRkTbgrSqTv", "proof_sha256": "2854cd27eceeacee271c4d085f98d15055beb14d608ca35c8a987442afc1ed1d" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954579-pmrTuHZEZd6BZwLO2Hy5", "proof_sha256": "5c5766c6dee33234eb417b88563e51edcd610c069a737e59237e894503d05482" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954589-1BzBdKpJK3ZwRgP16Del", "proof_sha256": "c48034593968e2a644f37a2747497aab38185aa23556041ba77166b06a78c0bd" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954599-jjGCPI8Znl1ofL7sRq80", "proof_sha256": "6b2b529dc0c0acce98d18861674d30d0c6c68e4004e07b6182a025fe23471199" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954621-xw0lr31s4PPloaeGsgEe", "proof_sha256": "f1ee044d8b57604073ed9959c79febb4f592ac7eb6894a5448126914c67f0f68" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954631-44H58xoZQkFJELr4qdYK", "proof_sha256": "ec8eee6da1f32e98cf3088a263e907a4e04e8473e98e80179f7d0a0abb07ad2b" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954639-EgP1TNCxGwsVg8QuUKel", "proof_sha256": "a6920f502307af116ca1e0b745ba0aa373ef7bfa8f6746da06fa08f9e07b2746" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954649-TyEFAtzZWXZu3AfrcI9l", "proof_sha256": "3a50ad35443ccba5fae0f2810000bad09be41bc76c825096ef6cfb9189d6c782" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954660-qNBOwbZV95V1rnireBEL", "proof_sha256": "ed79c2ae688e7c6166c4dcf18e116261a055fecff856793d7dce6751e24b7112" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954670-Xb2yxy1TNUWShQYVSEp9", "proof_sha256": "d930b7eec58868bdea6ba82d8a9e7d22a7bc77772150a13a7b80ec5adde7abde" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954679-ipS0fOrnUXP77ccQMjKU", "proof_sha256": "197c268f38c834874cafd4dba3baab87d2ef171186713b40cccbc13d3e2fe169" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954687-6j46a35Wg4YX3Km3qufY", "proof_sha256": "c403101c6ca4c300d37c2e5b50331e562483643c3a75878806e904ea4b87ffd2" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954698-AIZZ77isvzShuoJ0M4gV", "proof_sha256": "6374f75d4eb2b5d0c68b5223781efa66cd3120d08c3014555e814525ba1f7374" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954708-F3N04NR0qgkTmscSdICZ", "proof_sha256": "7afddb20f850ff201e3080a2c1669a4eb43597acecf344dc751c7437552322bd" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954717-h6Apbs1I51DyFEmzm2aU", "proof_sha256": "facd5c165df47b10edee97d076b2514e28221fe8d73715ef6906d841434a89ec" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954726-x7Q4kVCyrfaD0MaifNfT", "proof_sha256": "ae53d96f753412fc871c7d4f84846f9f92f9823c2b8e501eea7e1ae4320692f0" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954735-J2ChlFe9ceASNgULDPld", "proof_sha256": "0138fe1e389ba00ef3f27c594c11451cef472035911648eb6e29284063045cd0" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954744-KWQwmHYYtFWoXFG0Bshb", "proof_sha256": "3c4eee156c23c833ee3e390764a0cda34f59eaafd30c3496f1e9d2b2790f49d2" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954754-f2dbmVqxRDh23Vl6m4it", "proof_sha256": "d2d850e0417ffe92409c2cd92ad0df7d883b5bfdfda0d6fac831ea18fe88ba46" }, { "model": "inception/mercury-2.5", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954764-xCEbL9RHZNLkU0DKcCfP", "proof_sha256": "0134178b4d9509edd25201ccbe5804c32d514a8b39e313239639e4accfc75eff" }, { "model": "inception/mercury-2.5", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954774-4Dhh8qtrMrR2Pru2AvDF", "proof_sha256": "11220945096cac1833d5de4ed36eefd94721638792f4dc4fdc51af475da4a484" }, { "model": "inception/mercury-2.5", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954785-6oYPmKHbdsH4BX3EHrAX", "proof_sha256": "cf0757cae1a09ca4831e72d13778f811a20127e4dd9145fe3e76515bebc65297" }, { "model": "inception/mercury-2.5", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954795-51BAkOP0F97Xs7diQXT3", "proof_sha256": "a4446ba4192c864f57b2d98a10c3e5861a6dad3306f8deb91e3770275915f21a" }, { "model": "inception/mercury-2.5", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954804-sDEpRC4456KVZZjUv3qB", "proof_sha256": "1dede019c79a804def6abc9a73dad827c6fe3bfa97f043dac3b2ad814a141d64" }, { "model": "inception/mercury-2.5", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954814-uisZW0cGZfzd1fIrupCo", "proof_sha256": "9f91eefe0bfdf3a60f666952d838342a737e69f69ac62009a7ecd1d3088f40b5" }, { "model": "inception/mercury-2.5", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954832-NwFZRxNWvlRocSPA1ZY0", "proof_sha256": "753be31f839c14f2972d8eb994c88defad5e9d97c345a738e384524b53f35746" }, { "model": "inception/mercury-2.5", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954842-N1a1zWH1lyL5cPm47C2D", "proof_sha256": "1a78a65fc6e78ecc80777a163f4ae3508ef4debb9ddbbd8b364a708c97e9299f" }, { "model": "inception/mercury-2.5", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954850-B1MXcaBzqpvpfr3YxyGw", "proof_sha256": "36bcf2ad978d8688e44a84a00b92c31453f2d349be409674c1a4a5d498f41ae9" }, { "model": "inception/mercury-2.5", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954860-1zZ4yHPV3ndTUIBdifTT", "proof_sha256": "9a927eea2801988593287402ffa6f86edd66d73a765955013f2c95a490204298" }, { "model": "inception/mercury-2.5", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": null, "correct": false, "errored": false, "refused": true, "excluded_not_in_manifest": false, "finish_reason": null, "host": null, "run_id": "20261025T200000Z", "generation_id": null, "proof_sha256": "03889e5428f485535998fc417fdfbe53013a6c8321f49b114276b20d1543d5cf" }, { "model": "inception/mercury-2.5", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954871-WHcVC7Yybhidg7DCPTTp", "proof_sha256": "4306dc284171187939e2494d6d7caec57a2a84c20d6a428d6168a8a82a0c38ad" }, { "model": "inception/mercury-2.5", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954881-puwzHw6TieXt0b3N5l52", "proof_sha256": "b363ea74dd644a513853746f69177049d2cefa6c0ce6da02def7cf6f2bb1faf2" }, { "model": "inception/mercury-2.5", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954889-cMkkDT70rEBLsSu3GIeC", "proof_sha256": "249ab845bf35aea2b83581c88b873cffadea32ffc04161d3d1a2d3cf0695f8a6" }, { "model": "inception/mercury-2.5", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954900-4CN1z5IZSZWGgnZ6trnr", "proof_sha256": "483302d2a539c20e200dd24b11011ed62de7bcc7965cafed87337cd114b18d8e" }, { "model": "inception/mercury-2.5", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954909-mnQJBu93rVN6jJRHnhOk", "proof_sha256": "39e65f2e5022d5e58ed6c08d1681485b6daf8b2426dca34b300a702867669717" }, { "model": "inception/mercury-2.5", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954917-npIUrc9rn5QoGil6NC4f", "proof_sha256": "84c135a7fa526dcc7c61f3eed273764ed35bd9f236b84c2000cfc520463af298" }, { "model": "inception/mercury-2.5", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954926-CH4IKTQx1uxXPXyMEKfW", "proof_sha256": "4b2c97d6b6191f35d3cbbb7b63a16fb5f1f38b0b3f3eafdf23f41b83ac67386b" }, { "model": "inception/mercury-2.5", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954934-rTBG0Sohps2F1vKENw9h", "proof_sha256": "8f8369620dc8247c8d749c60e48ee6ae3bc9f5c8b59f2e085c5834007789676e" }, { "model": "inception/mercury-2.5", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954943-kO5iysWZBCqs9PZJi4yW", "proof_sha256": "da76567b0872f74ae14ef5746dab9eae32f237cca59ba5a31204d8f7c8321e04" }, { "model": "inception/mercury-2.5", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954951-NMnZGyr1hcIzVehDUSNG", "proof_sha256": "83d2045a8bbc2d5a18403fc26c259947adca20eac7fbcd9ebbb55f3722b71872" }, { "model": "inception/mercury-2.5", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954959-ykXCLfce3VPL5mVYr1Um", "proof_sha256": "12491a15228216442d75e760e68cb038d7f0777050a39200cfa656a680da65c8" }, { "model": "inception/mercury-2.5", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954967-AdjeOWeXeOlEinABc81t", "proof_sha256": "b73eaff28a2c53d4a6e87fa252966a6017e45c8ac21c6e083deed4fd196737af" }, { "model": "inception/mercury-2.5", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954977-H36KbdqsqfMInTpx8gyr", "proof_sha256": "5b14b574fbea045868c5981cff9a92c43474b964601c485c6e2d604446e1ec48" }, { "model": "inception/mercury-2.5", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954985-mFLFlpz7uvWOoPpKGOKW", "proof_sha256": "d245ca71887ea101c4e55030044aaa61869300350912adc32c4a1fe72057552a" }, { "model": "inception/mercury-2.5", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792954994-22XnSfBN8UPa3jj6uhmI", "proof_sha256": "25004be743f7e961b197554fbd2028b162683e46aac262c4da15042675702d24" }, { "model": "inception/mercury-2.5", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792955003-zQdQE29g9nqPrwnHKZUF", "proof_sha256": "e8c3b47ef4afd3a285da3821c008535a43d8faa6f113b2e7b740b1538624f7b7" }, { "model": "inception/mercury-2.5", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792955012-b9htwG1qqKJ9FdXYs3DS", "proof_sha256": "93520ff6ba228f8677f4e67f049a8b79758b0529be254a84b961ae1c021e2e1c" }, { "model": "inception/mercury-2.5", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792955020-a1dCPsVZQ0EyUIRtpTs1", "proof_sha256": "4645e0ad78bdaa66493e4b16a74ce99ea24fe2e9686c570defcfd2062e52822b" }, { "model": "inception/mercury-2.5", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792955028-j3CkCQ9OXrLUcWpLL189", "proof_sha256": "12ff7c9d55b6cc0eaf3e691c05119e1a300cd49a1052f4c57282877c9fef89a3" }, { "model": "inception/mercury-2.5", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inception", "run_id": "20261025T200000Z", "generation_id": "gen-1792955037-zwrlr4vIxOBdQ8RyHgmB", "proof_sha256": "2f364b685686b6d9e3c833c32cc1431fe256e3b86e1b91b239981a3b1b5ac5a4" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954256-sNM5HVGxDPIRTrz59h8G", "proof_sha256": "a847bfe15b3af0fd3cfde44114056f0db34dc89fb13dbc53bbf00e3a37a5d0dc" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954281-nUrZ7wsaMXJsN9lfXg1j", "proof_sha256": "b9700197c8d80b0010807cbe315e643c0275ce2bd7110ff39f6474e831b14256" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954293-12qDBf4K8Bg1j0nLeBk9", "proof_sha256": "f8703114f8f9a72ead891b15a2393cf362e2b5556d7d9a94a2811ba16847dba2" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954310-sK8N14N2emttFkt0xAzO", "proof_sha256": "bd72af4e856c17ef45aef60bb233894b525940bc57e4abe0b446f955e48b18ad" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954330-Sd3byrPe79nfk2dgA2m9", "proof_sha256": "50666b3f90cb7ea6e226ac5a560e33185a976adfd76ee34746c84b7b1a05a9c4" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954349-qKmsl0cG1rO35hBkZ4fL", "proof_sha256": "917d8d28e518b63ef5d037ecc563e8e1ba127848ae53dd98712c827d8b609e4e" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954369-04JW9hy7tVIG1VSdYYeS", "proof_sha256": "d8bf6a079f5b1bd3a566edb98129efa42edeaa438cbeaacf565b02fba9d47560" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954385-HZL3mXp2F3WulDwvQFrx", "proof_sha256": "9048dba07c6f9964dbc164b9e79c521a34853c3295b1f821caee07a14377210c" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954398-RZAnV0YwBoIvbjJ73P0S", "proof_sha256": "fdcafc153ee7cbdea34bf33289e9f67f659a021149320000e95a059e4830c381" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954413-aebxLKoOCoIiTjWP4tWV", "proof_sha256": "639ea0434a7be00440823d36fd784934cf6136e0d910ce3f9cdf91e73a99d543" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954433-fOtJxDgjtBFr7aXPauXQ", "proof_sha256": "292b1c3491ee6d04b6c9a76daf6d190260d6c1349c119d023b3ba23f3a46db07" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954452-IjPKVpX1JcXLLHKeUR78", "proof_sha256": "46c3fb90c980071ec777e894486e3a2437675a4192d99b241bc73623ba4ca8e4" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954467-AJHYn1QwUExHM5untnup", "proof_sha256": "8887a3733fa93c020f875b9d495351d23ee5800f2fcd9bed926b069995575363" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954480-Be8zdMnXwoGiwe9XRrnc", "proof_sha256": "3b666e0a3de91071a556e4fc88f8f5d1dce59acc26f7205a21efc8939aa60498" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954515-lsXe7u8xrcU1H2ob2YV2", "proof_sha256": "ae1606019ab88e773d91dbaf1b33bf03e6e261ccf9d527876c206b2070573056" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954534-oSoKkUQlZUe4qg4bjuuY", "proof_sha256": "360ac9bedf2384374e51cb475801ef9b86e382492fa85bc1edff6fd06ff08057" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954552-f9wG5hngLjmYUbpLrezV", "proof_sha256": "281662fc05fea78eb012be1a87e40ae6fd619211ffd5d1e6f6f2ac0bf49e72ee" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954563-b44reT33j5ybQMLMUIj0", "proof_sha256": "1bd33d875c40907de811cc236b12ee923f880c95e8c408a71ac6fbde4abc20f6" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954581-PLsG4LWzvLLo1pjeWSHo", "proof_sha256": "abd2ad7399a063f6f93b6d68f29b1d57dfecf82258329b8261a727dac4a8d28b" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954600-AQLyb7VlbxXPi9aWkPji", "proof_sha256": "9f4a90835db48a36c39d80ea197857bd213547992d7c618e30d61b1d5241b268" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "j (5 \\times 10^{17}) is the closest, but the calculation shows \\( 2 \\times 10^{20} \\)", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954613-ZAg5vJ175f73J7cm9VKr", "proof_sha256": "6d3bf53e258437551a5b448170675db346d63a264cc8f5b70c1622ed47d42b1f" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954626-piIRmfvtygOSV1VssBh4", "proof_sha256": "94f3d974dcbc20b476d9eba72943ae22653c97ed61f288efa2dd4866dc36f501" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954645-aTMkqsoMs5vS0xl9Ejvy", "proof_sha256": "2cca91c9700f9ae4ef8a09d44105da0c184967d2d76cc0bd0334d6ef39559899" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954658-h0g0QZsd1KumwagccIAx", "proof_sha256": "07cc7917301bb90cddf98529338cabd9586f88a2b99bdcec8d01b0745062d490" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954674-e4ECG7XHdjZun7llxr1V", "proof_sha256": "89041277b554a70b951862a2058e0c97f5b7832a27b2fb5a3f786264e91db6cc" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954688-SRkwAL64aGKxgKCwc95s", "proof_sha256": "3a4cd901adb6908e70f4dda6ae51d21100502595dcfadda2df2731125d1be18c" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954700-4dJuJ16qXqRrtpWRyICm", "proof_sha256": "1791ee0994364c9784d9c801a725a6e79319283e3046d9d1445b9e7793e88dcc" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954739-OYM36BZDSHE6lG0sXwS5", "proof_sha256": "abfcbbd845bf45e3e94cd58d962d139c68d3dab86d01ebb1be9ada0f3c1083e4" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954755-qAyid9b9WGHEkieZ2PmF", "proof_sha256": "d0bb72db07ab6a24f453434c026ce4e77ff0f14892aeb6881046fc858b2cb7c9" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954784-tUFP4GAEnMRSOL2hXmM0", "proof_sha256": "cead094fe834f43607f46408b00669bee37e7bcb25c3d781f6ea40c7babee0bb" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "\\tau1 = 1.138374 \\times 3.61 = 4.110 \\", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954797-LJPxgR1JnDhhlFsoZiX5", "proof_sha256": "1dbdc19b44d8ccef2c2ea2930521b7ab3359f16de77e5589430521b6820df596" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954844-jcI4yf5tuoURnh8nuXvx", "proof_sha256": "6702184b6add57b1c0dbb5d92745941c4a43160e0255b160e0810c1c32a7d01b" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "b}", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954859-Z33P1Gz3CHU2QVO9vEkS", "proof_sha256": "eb56efd3b2c5ec01fbb70b246fdc78453791f63fc3e809b796fc9d68009ce209" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954875-To0VqcmYfJSADlnsPb2I", "proof_sha256": "3537932be5dfb11396433a604c146d9292ccaeacf4a59782678c072ef6c92776" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954888-9CoDAKibAXQgpWvmyrXM", "proof_sha256": "d5b6f9d0d212ac71bbafa4a5e2277136fe0910dbf0da3ba31f368d5b767807fa" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954906-ZFU8D8ruf3chtnz1XEeq", "proof_sha256": "2fb6221bee109ddb0bc2a00b48f8d6631af70408e6184cb7b0f78adaa75efed8" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954919-6gxYcFHIN7aSCb7RasUi", "proof_sha256": "a7975813e9ded0ca9abcb14f25a6709a65c9166c67a737f8c607dfad5408bf45" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954929-SuHupR6RSrOWR9FuSpU9", "proof_sha256": "914d35e8e23ca0ffc49905500dd6402e1a1c46a78d7f0b4fe874ef8288c80e79" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954939-Bm96EBmYNnxN4zZjIIJP", "proof_sha256": "8511df4ae918d16300541d669d706a0cea85f8e777254b59e77186c6c43a5591" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "a (focal infiltration) - although not a perfect match, it is the closest option to describe the inflammatory process in the meninges due to bacterial infection. however, its important to note that none of the options perfectly describe bacterial meningitis, but a is the most relevant in the context of inflammation", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954948-ND37i70UHMb8yBZ1IS2f", "proof_sha256": "05701cd79c93f0692feb0009023b7f29c15536c46b2b8f5624ab4d827d2920af" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "c. signet ring cell carcinoma of the stomach", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954960-N0m51A4RKrQSt4jaAnTE", "proof_sha256": "28279b8a5b9ae1e321c4f9c7c1af0d9a313cfd8c7726948d7a36a8f6311cb248" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954968-UoUSJOiLwrei6oo0wgQC", "proof_sha256": "92f7ff8bb75dd4f6cf68fc2d014b7eda07fca9fd8ce28753e136b2efa955fd20" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "d. ultrasound of the eye", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954977-lVIhPJie4IhhpQUNoDlT", "proof_sha256": "2a8e8524bcde6b30d12d20f2e54573a12a50686234e71787e79d6317665ad5fe" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954986-ZUGMZVd8QhfcXijfOwE1", "proof_sha256": "1fd71f8169ec38fcca1090eef529088089dc460333ed642af2a2cff173d312d6" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955000-7kv7mc1XCd7JjtCVaTDA", "proof_sha256": "9b498719c6b5da6a802e04658b78e162a388166cd71b11b1112cfea12a7158e5" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "a. 4.442", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955011-M7YILjw8iobqEdpfEs8t", "proof_sha256": "889308ce6e8eea675560a8aff922a3586c29401dcc6e302122a55a17b7d42e20" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955027-EceSN7hu7ryw3H64JKy4", "proof_sha256": "76e9c52ee9df494dee422588d5c1685eb0bceca4313666df3f0a74c6d64a47cb" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955035-Gy86QjNniKrDF0DD6Gz0", "proof_sha256": "ef7dd9b583223d3de9f50512159aa8f31f6540930b43389afe896e04224eff81" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955059-wu11KzoZwY1Y1cbfpKtq", "proof_sha256": "eec51d0ed2862b75cb0f6e661a0bbf181d72a7ac5fc8acff694a004ac97faa62" }, { "model": "microsoft/phi-4", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955070-c4vbXnEXmuXX6644Qhen", "proof_sha256": "ba92274277c56b07a3753d14a275bd402fc0af25d9291a05af765df30668b5f9" }, { "model": "microsoft/phi-4", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955079-gXblQI3487HY1r1EGR2K", "proof_sha256": "fb68acbacb0c9c227f4550d01814ee6cba55ca7c4594b97c9e026f04ca7cc1f5" }, { "model": "microsoft/phi-4", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955092-MfQMxaCC7DciEH6Lwa3N", "proof_sha256": "61eb357ae0733cf2abb8e8df892bb30780b424b12f81715bda687eae8f182d79" }, { "model": "microsoft/phi-4", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955103-AETIBXdMAwrhh6XZZepX", "proof_sha256": "b91e4d9052e8347f441262b9b1e6b23b4cb684fb694ca12ed43fef38a07651fe" }, { "model": "microsoft/phi-4", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955112-YyRduhqxZqgYM7tmHaae", "proof_sha256": "15505859ec19661882449d6bdef20bdca5807976f6ffadefb6944074975908e8" }, { "model": "microsoft/phi-4", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955122-CUO7T2dtlSTrjU4QmqEX", "proof_sha256": "151b68397c7b9612b580c0cc29e80e34e3430a143ff0bea3d14a0da648d10d39" }, { "model": "microsoft/phi-4", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955130-R0ndV2AufHvlltF6Kaqp", "proof_sha256": "7620816b1bdb089104672eb0abbcc1e8e211a1de686c22e472eafadbc542f796" }, { "model": "microsoft/phi-4", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955138-EEXuxVJU2Z5VI9Hs3Bn6", "proof_sha256": "45c70c3f82a28617c963acd841edef75e864c257cec87b6002952f91c75ba8e9" }, { "model": "microsoft/phi-4", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955147-AEr62gvWivz5oT9eCdua", "proof_sha256": "61ff26d1b14f65ddd8016b9a6761082b7509f9a28149b90e0f43a3a5fae8483e" }, { "model": "microsoft/phi-4", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955156-uJBcrDM3RBx1QU93CksQ", "proof_sha256": "f71a1e5e9f32a097d5c4fd087b0316afc2d82ea02786a45305f71a09381e5d1d" }, { "model": "microsoft/phi-4", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955166-VB7xTrK34QQsq2hqXyHS", "proof_sha256": "3d833a6627f4f8a52c917c2656f001bc2aef561fa10ddf901a80ddd9c7da4226" }, { "model": "microsoft/phi-4", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955177-jK8jCceXqTY4JOqHCElF", "proof_sha256": "1bd30c63b7244732989b8f851629fe26924c37ae0a51851885196f7f69f8d5f2" }, { "model": "microsoft/phi-4", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955185-mZENIcHLqCWCkUlNQeLu", "proof_sha256": "eb033242ad2f3ca4ecde1c250bf6c61533bc38ecb7d0297f578afb97eb455cfd" }, { "model": "microsoft/phi-4", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955193-6lVcFDJBj1Cd2RG5hLGC", "proof_sha256": "38ba941ce83459b137b5eb11a62ed108b7ada25ca2ebef05ffda1e93b329f53d" }, { "model": "microsoft/phi-4", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955201-0SvfSPieGVFkgPe12vka", "proof_sha256": "0d6c10be0c97795dc29234a9104f388a31282a8134afcdefae4fc814d3df23fc" }, { "model": "microsoft/phi-4", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955210-QbVqCWah0UKRaYq8XnCW", "proof_sha256": "e59e7e4480e503670c6e03bb1d8b74b4f517403ff7c698da431bf747a6db19aa" }, { "model": "microsoft/phi-4", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955219-ZpwaQniOk06RPeLBdgVi", "proof_sha256": "8deb9578599eee139ff27714915b62e3b5104633f194f5bf0aa86a81c54e1119" }, { "model": "microsoft/phi-4", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955228-J5XOp5Dg7iuqjloYSzKV", "proof_sha256": "1f199eeb9a30e6c7763b8fba0bb2d04a52c4d05acd0d089d25568f298327ccca" }, { "model": "microsoft/phi-4", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955237-vTL6OwQ3HDIlsw8SoO4K", "proof_sha256": "87f06bed78eae3e1a9e787963b531840162324c1d680f7312afacd5235edde34" }, { "model": "microsoft/phi-4", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955246-mpy9nLz71Dy0FfRQw5I5", "proof_sha256": "ffd34ee970c0d9f1372623b6591aba27de078931dd65c9c1e2e638cdb1b2d149" }, { "model": "microsoft/phi-4", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955254-cPbWIFdLOCkKfE8yWfRp", "proof_sha256": "bdbb75fd48d955e8c689a62e86b65c2cada52684bc13da9c119a8ac28c251fb6" }, { "model": "microsoft/phi-4", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955262-CAfwcRtWUO3Nt72PkXRe", "proof_sha256": "25029c84cebd236852d1bdb9cf82da829f8354df766206526f6531ef1708d900" }, { "model": "microsoft/phi-4", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955270-UxLF7Lyygx78PK2EICjf", "proof_sha256": "6de385dbaec6fb2e7c56a702c55657422ab05f3c7ceb7da4d8acc6042eb437b4" }, { "model": "microsoft/phi-4", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955293-AF3Yi8sffCifXqhYrc4V", "proof_sha256": "c4d96f56519e978a3d9ed08cbb5dbf6fcb5d0ca3f75bf7a8de386910b3d07475" }, { "model": "microsoft/phi-4", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i. \\( 1.6 \\times 10^4 \\, \\text{ft/s} \\)", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955304-jZkcnXqyYAHLoiaVaGID", "proof_sha256": "7a43cbea1a4f3b2cd7185dac6bada702e078448fb329a2e0fb7d19ea319005a9" }, { "model": "microsoft/phi-4", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955317-uSgXGCGBrhOPSbzhOIMD", "proof_sha256": "d157229a49701aa72cd07cafb062251100f57bbd72ace4405e82a45e0c765361" }, { "model": "microsoft/phi-4", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955361-vASZevAe2TcWdGgHyZW1", "proof_sha256": "03e62aac455d388af1f586518118b6f680c8ac1486e24636774dd5cf41ca0ab0" }, { "model": "microsoft/phi-4", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955401-q7g0aNYeTbLsgbWNW6PK", "proof_sha256": "1f4308415ffc2326d2f9a2fb2e0cec333b6d67c73201150c32d4b66daae87289" }, { "model": "microsoft/phi-4", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955445-2wFwMlwY8C8SAqBmyj4g", "proof_sha256": "e1cb91d46ae33224c1453dda5a489628c16f2c40c1eea205f762add3d02ff64b" }, { "model": "microsoft/phi-4", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955486-7dpOUCXFlFcwEyvrVJku", "proof_sha256": "f9814c527b8cf7fb327d4fe2c4f0e1cfba27b2281150bf62b6f78994c4c35b95" }, { "model": "microsoft/phi-4", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955525-30XSG9qXIWSGVfZETZde", "proof_sha256": "0b2028536d2eec8447c3e86cc6dc77a0e269f3aff29551739489753a876c98b1" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954256-hpUKM0wL5j4pP4TcC8fk", "proof_sha256": "4e66b6db5e8c27f1e4e2c116030eb438f3abd9c621fecfdb5753ecb091a138af" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954281-bHCIWILKLUWwXhnfCA3F", "proof_sha256": "8bb90279fac517e9087942641de6c242f897cd25d463ea5a2a4c7e815bcf3aed" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954330-3ll1UibcBpOq8ShrEdfi", "proof_sha256": "4cb0d6d4a14c89af7a0769a444944d9327cc7bb9845c0ba5bd40f03e35ab20a4" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954372-LuCWFDG3QxjkqVSGBYwA", "proof_sha256": "32287766924e20a80b0c5b5b5b84a8d84172a215567fa09e7028c33a50232b21" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954404-fiUI6k6Rlq4oIhbBFvCg", "proof_sha256": "41d0dcd7f4b77cf4aa64d60d820139e49e447a3e95d874b30c6e9a5b4c69da1c" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954460-71ZtEBK3vTyzr7lA9XtH", "proof_sha256": "59e69c11a8f35e679ecdcf2d04b04225b2fe1b2d3f9d105d76414cb24d68cec7" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954500-Esh8Jzzqmn8bZ1SWuZWo", "proof_sha256": "6b26665eec7169c3d94ed5ae6cbbe8ddf0f4905eccc052ed33f758ac680cd6a5" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954527-84TU4BC4jkYUh8ZeaugI", "proof_sha256": "d04c2fd105cccf77b75ad7c669f94a33cfe70e7fd8772b2f0256d02d8effe4fe" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954567-WVEpeepw8YLc0BpLwSys", "proof_sha256": "f41a61dede7ed6c556d70038680f4b80c9b8d165abd3f86daca506ae2d9c41c6" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954631-LwppyF8ynlqlR1vFRkOC", "proof_sha256": "1af97ead6b4670d8cb3253168b4b815359a56a0f7d241ae0fec7840256f38420" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954672-XupigenFKuJOF4kwxLgl", "proof_sha256": "404ccc8a0226b811e6fb83a239d02ba68e4537376feb54348ba277904ba0a622" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954715-oJgFJVASaoubYBmWsYpa", "proof_sha256": "f1cac93d9848c6cde1d091e99f25f648bca9f84ba4da10b753b201c1ac9a49a5" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954768-RLhCiFYZK1ycbS4iWvLa", "proof_sha256": "effc4209e20f70d15a3228facec8914329ecd48d5161d61f7f3a692919d6ff31" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954780-98UGefEQCaTpW38R3g6R", "proof_sha256": "e155a4f67d64a9802e50b255804857c83f68e8aeff398fe3e8288c3d70c0fd39" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954803-wipNcRL3fcpK4B2QGAK3", "proof_sha256": "5e7bef6a5dfb834e746b365ab8c74e1179ace89c0cdf0462c5f7eaebeeb5f3cf" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954843-6R1zH6EcHtbp1FjjPHvQ", "proof_sha256": "f428d4828736024bb833ae7bd21f681cb6fd82d85eb5b1443d0e615e9d0f9ec4" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954863-I1WlopirYj0LLtRttxu9", "proof_sha256": "bbbc1dfe39079b2d2abdb6d4eacb0309998b1d1b460d79c2fb18f91e99a2530d" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "g{\\mathrm{agcl}} = \\frac{1}{1.00\\times10^5} = 1.00\\times10^{-5}\\ \\mathrm", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954899-5m8rhW17qU1NBtgJlQd2", "proof_sha256": "bc31f471d09d9a0e5a0394bb836a2e4e198fadcb02ea3b9528da0504857093e8" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954942-gKVEN3p8CMuJ56CIJfSz", "proof_sha256": "8ae9beac718cd7a8d5601aaaaa6a43508425e8833fac392a277c5099854cd689" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954960-hGX3TyTzaOgM81dSxH6v", "proof_sha256": "ffb662db993a9ec5c1be1f1b10eb6ab2a424ab5c3720beb6514d9bfc0e174f79" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954974-GhEckI7JhTDyIt13j6Wh", "proof_sha256": "a5494c0b2b9714d5ca575adc8d106b8199244144d59ae4a46391ed3af248ab79" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954993-wmB8W332JQ3mxRcz026H", "proof_sha256": "492f481d40b5fd316ecbd72f6debae180bdcc06554bdbc806d7847cbec6900f5" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792955032-LX8WJjQgWdGLfm09Jpja", "proof_sha256": "ef15c0957994e278c0923ed169666b7f60c1b843127588043739a3a7c0c3bdc0" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792955079-07XY5QTvRsrHIPUbF2yO", "proof_sha256": "01d82113faca35895147c0c4445189cbcc9a70c9efeaf2fe5bb531c36af590be" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792955116-NqEK1LfdgH17yLOn7mHj", "proof_sha256": "8f1dd2e41606a85d2d4797ab851f78ca3374462a0f70d4683160155ba12cda93" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792955161-xPlXYDMx0Baa2CpumNZe", "proof_sha256": "1a4531c9b0e82a42f5da2f680d8ae5766c6217f301ec5b141f34fa5b7c10b816" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792955202-XtM1pOMz2sJKBLqcExyC", "proof_sha256": "407f6e94b8e44b777fec3bd580dfab693a3539db4d1dc2ac7a75fb3f223f5b64" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792955243-z1IlAZgEHYDbp2F9rZDb", "proof_sha256": "07c46e7041b7e35c0bd44119ee44a1e6e8f760c9a443d4e8fb961d907ec2cc8b" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792955286-hHE2ug4BgW1EsyO2Zg7o", "proof_sha256": "3456346141e7cfaaa47b7481b70d1982fe5a09af39a2a355cf1033007ee2fad8" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792955335-oEgP1lV319Ra1mHOtLZi", "proof_sha256": "6fdce18a14f5863f5b3328808503400264b9103f016ffa24f8024ef50c723a49" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792955414-c7xKa0B4RLcw9OET4yUw", "proof_sha256": "22e09231795c2c685d2528fff77f8f4850bf08c3344a0be1d8784214f8c9cf39" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792955511-JPjfWADTsky3PeItdFSo", "proof_sha256": "5cccb6294e1f666659d2320a44c83acc792b88d771afd7c0a3597ddb5ad286f6" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792955558-TyLc0DNMth6Lbmn2p0xo", "proof_sha256": "524166f4d8ee370a5a997801c9c11b678926907546b638e819f79859482db4a0" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792955627-UbxZ7w58tgeeCxX7sq6C", "proof_sha256": "de4cef126d88559612d8153a6f1385cc1bb096483dbbfc3040ce4e0ed0ba768a" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792955699-rvKSWV6wl8isjBwPAoa0", "proof_sha256": "cf4d05ce9f9d335416ef9ace9fb843034e120c6114942535c5c7fc485203fa15" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792955797-WmoHztucNWKsrXjTtLVf", "proof_sha256": "f4a6253f0b612fdc867fa77f36d0b592eec7016886739fff07b84b519b3f8790" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792955854-LOIg0GQf37XFGJNakW31", "proof_sha256": "3f94846fa33e098bbbf4543622fbe7981722e606e8b918745d8b69e6e0db9b9a" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792955924-H5Xwd0UYNtkmTYSz88pz", "proof_sha256": "0fa3dcc565c8f4bba9459c2090b88b3d6d5a32b86e9a63428da6e7bbd9457719" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792955978-Eu4LsmEJUTt1JnGD0kHH", "proof_sha256": "864c02e66fec29687af075b89d63bfbecfcecc6c6d544f0112469e207552bb6c" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956022-JpZbiXtSzjTE1EUoboA5", "proof_sha256": "c9122f72fb762365dd50c1d508bf8a97a7cddcef69f03192a9990a22de279bf4" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956071-Y2lzE20iwhcuRaRIKz0R", "proof_sha256": "2f5b130a05fe658931bd2654f28236071bb8b7a31a0fb16bfa6ce03ed48c737f" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956093-TuXhbifv6f2yjBv6aIs5", "proof_sha256": "63f7504c2a0096d1e0520bd002fa42de60e1ee8a8f1e8f51a31b6080ea26445e" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956133-CVRn1CXTAskAGdckwnTw", "proof_sha256": "756a4185d44e52c942d6bf2be84ab72ba27d613fa46728afb63eaa69a54bd162" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956149-rNRUxLvwlGAcq4BiXthz", "proof_sha256": "9f63b4babda260241de12a37d0dc1495cff71fb90ac9be1951ee209171ed6022" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956191-veVIUtuh9zTcvUj5Lx0j", "proof_sha256": "46af9b14a03e0b9a3ff1935bb7df32775c6ac7f7ad477789e6fbcc7fae426c8c" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956202-1vCj5f9uMtaxIuwKEGF4", "proof_sha256": "377bb65f020f9a07bc7e2f7a58325299b71426145d23ac4321cbd194ed51ffcd" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956223-BEnph0gj3m9ZrxEs6D7k", "proof_sha256": "4c365296d7b971563ce4c3608efda381c7a1ca9ca8244a299e566a5683f8de28" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956238-PahXhK12zOJmR8X7IPUK", "proof_sha256": "489803ee5d4e4306786f3b66b8dd8972cdde33aabfb2405cf1421e8e959b4166" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956279-3rUf8zOdcfO0QuUSVJIN", "proof_sha256": "ee4a8ddfd26ebc45713e62ed9b54122c4924278cb02f737f816b74ba22ea120d" }, { "model": "upstage/solar-mini4", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956329-yxoKSkOqkkJYAcvOmvpP", "proof_sha256": "5d6aa1c61c6bd9ed795293f3646d83829660d7131bc4bbb10d47271547ebd6af" }, { "model": "upstage/solar-mini4", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956373-HcAcHN1tvo0CUdTfOXfO", "proof_sha256": "b13c385031f353b51a1c80a268ee4b96bfc55c5c7beecfc850c0bcf9273a8dcd" }, { "model": "upstage/solar-mini4", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956424-PAil0XNJL330aGZVqLUN", "proof_sha256": "101165b1d48c38108640ab03fb90bd5dd4d4632a3f074aa042d203521c89b2c1" }, { "model": "upstage/solar-mini4", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956501-lwrJ6IFvyfWTkvSAXjHc", "proof_sha256": "e21afeddba29d0a4088b03427748a0057e765bd0d98aac6a77290bb6337c5382" }, { "model": "upstage/solar-mini4", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956541-K1HPhOfWVb5BbPnoBPgX", "proof_sha256": "564a4c0308d9475b8102e507d298341cd665d9e39064acaad20d7bda7ce1fe9a" }, { "model": "upstage/solar-mini4", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956576-ZDdLp2ntnHj9xbFoLwC0", "proof_sha256": "6b1f107fffc6e241c8ed88bd27b871047654fd32d08fd1eb770e44a5b21359f7" }, { "model": "upstage/solar-mini4", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956624-o9YvauBgn5iPsbprmqqS", "proof_sha256": "87984203a7ec7bcb97f175faeb415c91a87d69d93f5be6fb757fd05761ce21a8" }, { "model": "upstage/solar-mini4", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956666-tvomP6H7PezXCIi8GiiM", "proof_sha256": "e82882e24dc8b95e17ee7c917705c87dbf7a1d808a9b39e9cb2c54b09f8c46ad" }, { "model": "upstage/solar-mini4", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956711-AfytwFhStU7tqfJnxtiA", "proof_sha256": "840744b883fe6be6b19474f153ba8b6ecd051cef848a36db914c994bb50d4c47" }, { "model": "upstage/solar-mini4", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956724-0cwrI8MvM31IWMgkFnuY", "proof_sha256": "d87f3a4a1db7944814bed9461de016725700383694e9aab0d8db65795e635862" }, { "model": "upstage/solar-mini4", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956733-ZRxAnLJczTvsawLFnqCM", "proof_sha256": "f7c822c075f29a704d49c6feec4104d914f7e3762fceebd3d8e5fdbe8273b913" }, { "model": "upstage/solar-mini4", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956746-Jj4wrweQqvgQLe7M5lZQ", "proof_sha256": "3d4424d4b2c8f340d4034381cbd9a7bad999def93d07cb60df404790c6e8b251" }, { "model": "upstage/solar-mini4", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956759-SUgz4A9XV5SySQ4nAgmK", "proof_sha256": "f1e008c285b1b91aa60c2a30b9dfd67591aeaff34f59135385f47401b18352dd" }, { "model": "upstage/solar-mini4", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956769-wTH3036GA705z2gMydmo", "proof_sha256": "5e360759030f022450e560fc37fecdc44469e3ecce9291ac0e6bfb2969225945" }, { "model": "upstage/solar-mini4", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956779-do1nwoxNmrTFvwRIMegL", "proof_sha256": "2cdb1f4e65f38c753c61afc4b2e5c794f9d7a35c9b5f0b47f3a783bacdaa1b3d" }, { "model": "upstage/solar-mini4", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956792-RADnQuZJsTJYCwOMDCqy", "proof_sha256": "cdef375ebe5dade7181b0452b34adc96cd8e344d2fee5a9112fce945be374898" }, { "model": "upstage/solar-mini4", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956814-av2PC0U7swzP5rg1fZcj", "proof_sha256": "a9a42f07f3342aebfa8db2f4466381cdbd6a581f3f10c9b9ddae05a1fb11526b" }, { "model": "upstage/solar-mini4", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956823-s3jFWtTfjKbItYKR4IKq", "proof_sha256": "8c1186500b2b2166a0a5934779754afeafdf9d511ab79b20543995f220cd6c02" }, { "model": "upstage/solar-mini4", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956832-ge0HBfR3wqy5W2yQzhuC", "proof_sha256": "cd91f8c6b7d3d42f227d0f250947aaeb2b526754251b56df37fd80aefcf3f29a" }, { "model": "upstage/solar-mini4", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956876-d9mgSKLLYUV8SuHAQpWe", "proof_sha256": "2a4220fdec09e27d14aea3bdba8634e9e647c6220ce43a4fbf656ce438cf881b" }, { "model": "upstage/solar-mini4", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956894-YsdNvHX194DYpIgrEoAI", "proof_sha256": "0ccdf96a234db8d6921836ef2b84062a0ddf36665a56fcc76b43c79a54af500b" }, { "model": "upstage/solar-mini4", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956903-9RlsBsYvy6O4wJkkBCrc", "proof_sha256": "361935fc731e26c5fcb8a258b85f5667ea1834252665a4f139355e6b62272b37" }, { "model": "upstage/solar-mini4", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956911-0GtJveYuqpTkj6bnoVve", "proof_sha256": "ead5d6d4b8d2e0e54b6c9770e8f30aa33922ec18122478a16ab0da42c1fab93d" }, { "model": "upstage/solar-mini4", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956950-qZRVIciHE3lIhYiIt9y8", "proof_sha256": "6fa1fb0bed6990f0a0eddec48d1f8871726248c22f057fb4f95f1ce5bd848d72" }, { "model": "upstage/solar-mini4", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956961-Pnhmwb9cChMcVMWQD4JA", "proof_sha256": "b19deb6f884f0acf5064db92a9e9c3131452e35ee2297bbd85ce02e0ac6320f6" }, { "model": "upstage/solar-mini4", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956984-2SRWrfunL2p6zmkejySG", "proof_sha256": "af25b30574c06861b0cb8d7955032aeba6fa90d58c4e46a49fc2291af4280e34" }, { "model": "upstage/solar-mini4", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956995-2GZqIGuwZ9mz6ZualDQo", "proof_sha256": "ed7040a8f27131db4501409921de2efee0398f06e4992ed060ef4b076f4ae356" }, { "model": "upstage/solar-mini4", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792957008-A4rQfamAU6cbTqbrBnuq", "proof_sha256": "8cca2ed3b25347855f970f5c50d5a39167d2dfd4eea33d01123efca16919f8c1" }, { "model": "upstage/solar-mini4", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792957057-FnSVo2ogBrSh4EuYdeFa", "proof_sha256": "799faa14c09d8cddac6d7a4b56d56f989a1f4a43b46a90f50403ee989d252a26" }, { "model": "upstage/solar-mini4", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792957067-ZJdqGLULCPE3pgr7AHAb", "proof_sha256": "26e38a912a2611a1ea4858266db8c8ceea0d6884ae6ba689f4a633d3d2b11518" }, { "model": "upstage/solar-mini4", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792957076-wSo8Vp4pNrLMBJ6qTRCh", "proof_sha256": "32e94de11ea0169dab404a8695f7497710b2fd633ff32d5bb93f812419d915c2" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "\\boxed{f}", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954256-JxCSs9n3pWEhVk5wF0XI", "proof_sha256": "c4cf926dab88dc409e09b5e9504e05d2eb2310888d6d4c6903630f5e0c96b0ca" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954286-7MdRtwJ8KTXZVsIOvf5C", "proof_sha256": "747a6f3c1277a2749c4e6597cf7398fd8366aee131b4358d7f385308eab8b63b" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954313-zm2uIPp5nqcytvvvbX8E", "proof_sha256": "97c56fa7cd393a457bb1125ed8bdd2acda00a76592d65a51ac2cc0d737acda1f" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "\\boxed{b}", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954347-95xXF4nC8AbG8jSvsJ01", "proof_sha256": "bd25cc97ad623cf6bd52256cef66e6a643e599688e25d911801be1fefed57497" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "the term \\( q^{(1)}(", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954383-ec3EpRzi7cOsBJGqDAGY", "proof_sha256": "0f03cff1d6913650e20d59558966079ce8b44c682a0209c690e6b624e957ced1" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "\\boxed{i}", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954461-DKqCPNDqAEHV0Q92FF8Q", "proof_sha256": "cd8c28f38a4fd74f6d880dca9d9c233cfaa1c4c2cf3b5a58acc055ea5321aefc" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "1.8 \\times 10^{-5} = \\frac{(0.0", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954512-MeeOae4ggC8Ck9C9ZyvV", "proof_sha256": "72818651c8a54f5fd2d9df8122b3af950664b9ca65f418bd5250e9d1c8c5a671" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954597-AxyYO36gig8F2yyQJ6tZ", "proof_sha256": "ab907c1a051068397fbd8fd173c2717811b17dff60edf16f568dcbfb23c3e2c6" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g (assuming the question is asking for the solution in the form \\( e^x = c e^{-x} + 2 (\\sin", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954623-n4Dqpy3qy7Ek9FqoXBna", "proof_sha256": "88ae7804241ca2743a455f13643fbaac45fb92135d9719cff1a1594a08411918" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "looking again, perhaps the expression is $\\sqrt{\\sqrt[3]{4", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954667-KoenJSdlq5QnCrbLr4P5", "proof_sha256": "7471e560c7ceb034887089ae5e5d7f85dc2586fa1337a0dc40c86a45b620c07e" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "g(n) = (f(2n-2)", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954788-ROZcimAyvwrwdp3EmFnY", "proof_sha256": "477b8405721cc4e1e23ce1d59c5326a096441bb58844c3847745210c5d9ccff0" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "\\tau = \\frac{30 vr}{vr cr + \\frac{900 vr^2}{31 vr}", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954844-r0Ii8Mz0HoHApPznNI3F", "proof_sha256": "6f84048878e4217723127bcd23fb2f9632172a22c9c4be879b16137b2cf1c8f8" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954886-qWXTpaJA0SUjypZWcCqu", "proof_sha256": "860e18e8b64857a3870f70571a4260ed87fe62b6638828a535e7f268ffa5ece0" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "answer: b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954896-1AxA1qHubmC3DR6you2R", "proof_sha256": "23d7dc285eb476b13961a01143966516477e19ae1ea292643493d85df82a4b7f" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "- \\(2880 \\times 76 = 288", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954917-4f4VZJ37zXQzwXvK0y7B", "proof_sha256": "3573942133875faee8be3b8ca4af7eb316da64e12e8d42e6c4ba4efd619d39c8" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "given \\(v = \\frac{d}{t}\\), and \\(d = 5000 \\text{ light-years}\\), \\(t = 780 \\text{ s}\\), we have", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954952-X0Ck2lFVTSD3XWTSFpcK", "proof_sha256": "cae05c7759231fbac783d0340992de1e1ead7667ad95e2ea9084982ddee5d4ea" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "answer: b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955009-x8wJDD2BX0227sMyV1nC", "proof_sha256": "10d7b4fefd58a4a08cc50e521f08200914741b34feaf7b10188b83ff0e8bfbd5" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "the closest option to \\(", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955028-9anRdTGkdRtu97qaW75y", "proof_sha256": "cf0889c3cbfeaead429eb42ae70073d0e7b4a67a614317d8322edc177861a5a7" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "\\boxed{e}", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955073-N7bctwAZ475hKri6Yskq", "proof_sha256": "3c6a7a8cb2678709c754d622318c16f076a289eb98bfa6aeee892d8612c85602" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "\\boxed{i}", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955106-QaJhrWD4S0078cOM7Qdv", "proof_sha256": "a71d87a1ec5d0a26b41a68efe97d3e306b77e619c1a34f188dda66d197bf3ef9" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "d}", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955123-5aNjzxsBgvbwvyX6agsY", "proof_sha256": "56dbbff302b9f31a2f75617827f7a189991d426942738deb4ff9eb052f4455a4" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "### using recursion with exact counts", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955162-WS7RsvlQzjUUcSXV0Tlh", "proof_sha256": "c3071d22b3c48e9648065c8ee4015138ae03ae326f4dec5cdbf705dfbbb860e9" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955194-gyii2HDPAroDpTPbypFf", "proof_sha256": "571c9458f35f13f82422df35e6f8150ef40a4105f17526a54a30e0083ca9ce1a" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955217-V8Dlbrtvhmimiyv8K2yf", "proof_sha256": "95d09c4e69b1734a17af07e03bbc05c9f0378892babf7886cff7e27d5d52c2bc" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955230-XV7j3fpAS4KJIDXl1rRk", "proof_sha256": "5ac2f79163d22410ca6769bb2bc847894b2e9cf5649240790e043b693961a229" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "answer: c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955243-GYeSu5rR3pbOA6nf5uRB", "proof_sha256": "e9a5ee8e1349b4ad446760da0f543d7b931e5ec9bfe3e9ec63595d3f64035176" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955255-3mxEorvv87hto2oV52Rg", "proof_sha256": "47e2e8bc3d609903d31419a4b37e0869be3b81510efb78974059ee07733359b5" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955276-P8RdMBjf1L4XJmKA9DEC", "proof_sha256": "e8127736b099f883c0987f992c5837683f88900fc71a27e02d2fe62a719a4920" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "\\delta u = n cv \\delta t = 2 \\times \\frac{3}{2} r \\times (t2 - t", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955305-Fqhi80aZorIYoUnXBqNw", "proof_sha256": "c7e1d6d692eea42ab66b18e6917e62133c9208cdf11ffa46abb380d2142ad3c2" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955335-KAZs4CJTKK0S5P01y9D0", "proof_sha256": "ce1a628dd113c8c5367f5ba5a4829c474c2c5eed53ccf40e70d7d9b98077d0ec" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955392-QAOMDs5GvdncSx8uSQqy", "proof_sha256": "5da2651633f997b18756d9113e09b162199aeb65cd805d912510b83aa7a754e2" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955445-t1WBRyn2EKpvupVMT06e", "proof_sha256": "bd4fd9a33010abc07f8929e6a824e01eb9f222ee01c826f24f5321a754f39719" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "ans: d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955487-uPb2oZbYO3b5dCRu5bwz", "proof_sha256": "e0ad32c9664b04a5e6aaa26de39dacf9e7b8a8f3e7d5e25107bff9c6082ef8c6" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "answer: b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955542-uMXQ5OMrUYprSu9RmrMr", "proof_sha256": "860bc265ef90ec0085ce370ac41bb2f836e63772db98a77668373b4c32c1328f" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "the acceleration is zero when the magnetic force equals the inertial force due to the change in current, but the rods velocity is constant at", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955583-qW10MxHoJmqWtNVJi8Ju", "proof_sha256": "1d18eb771b68daf6f2c7f7540e7680042fc5854e1212501badb420ef3fa232f9" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955697-ghE52tmL59O3CuHQibKI", "proof_sha256": "0f02f438e9528552174bf19377d09dc2218d4cde9325498487105db82ab5db89" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955797-1oHzGW50gdnn6Df0zPqV", "proof_sha256": "cf88050c7614efeb98a1d3bb87813399e29d09ed2d44e470ee4ce413d2af165c" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955842-DrfTjPpkrKJbTdmXn9ZJ", "proof_sha256": "df133bfb34f63702ca4a66af7d4227073781a4fd9e51260a87659c25f119d9ec" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "answer: b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955856-TYyQI6ai1FsJQpbBgdC8", "proof_sha256": "e1e5f347eb4644e3ea43ecd4b591397cc11db441379c905cc0a8d4bf1df1be05" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955899-N5ZShFrYuuPkIJ9PmcUI", "proof_sha256": "257fc9936dce0ea235d914427890e0c910f62f8e995a7a85cde55c021e72a8d3" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955953-CC6NyZfWTgQBNW5nL3Mx", "proof_sha256": "4ad0f02bb5bf39758fc71a50b78d41eedb415d06a0afe8d697168892b351ec17" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955994-jGaC3nNk1OcqTj0Fwqn9", "proof_sha256": "811b609164a55c944fe86caa0ad8b1e7394b5b5152e808289782ec5c81e4cba5" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956020-IYW1CUjAQXcOc2BdqQ97", "proof_sha256": "a64313ba35c15efa94e5b32a5d68ed3b143e3fa9a4ef37c901c2b28bbcc13abf" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "- the enzyme is 3.4 mg, and the moles of", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956030-44MLmZ4QCqWR7KyXbutK", "proof_sha256": "e5e729c3dc8e270deee3cc6ecca47dd8efe440dd3add20701a2263c7c25bbd14" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956108-Qof1tFggWILQnec8nBm7", "proof_sha256": "1ac662b6fb8bfb72b644bbd250eff1e8057daf6507b52108f3b9e76d72fddd80" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956122-vnSR06H3uQuROqtwaKxV", "proof_sha256": "2fea0f536cdf1a6dee70ff3925b48d4533d9f6965e294f12c30e43e325c39304" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956143-qmfyoLUMKfOoDlVgFMUy", "proof_sha256": "5b454c4c628ef6a51dd34c004597f0b6b213eebef515e2b3eb1b3d5a24033dd7" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956155-blDN8lSVIuzwdBbD1aYC", "proof_sha256": "08155690ea4d3c1ccb64b36bc0e475514e84d0ac0c2595d58254372c6df4ccb0" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956197-N0bPseDeJ9liGsQcNT4r", "proof_sha256": "c6a5024a10fc73301a257c092b6be15682e00ad6f88f3577163b11e4464e634b" }, { "model": "mistralai/ministral-8b-2512", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956223-pl61PAOR6KgpLeMJ4sl3", "proof_sha256": "74fb0f1df213ed1b99d8d7d58abaec24c88a43bc630a7751b5db3a177c5dbf0d" }, { "model": "mistralai/ministral-8b-2512", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956232-PddbOf8N5kB4dThk3vWt", "proof_sha256": "00a31cd4030c1eb3c7ca5bfcaea9c33964cd562258ec26f7fbb8636ddfa8b55a" }, { "model": "mistralai/ministral-8b-2512", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956241-YjlqWvbVVxS3ZcokkMcE", "proof_sha256": "ca8cd12f0b730111d0403e6de1e19aa384cdf6df17158b07e1d72c3ae2b4300c" }, { "model": "mistralai/ministral-8b-2512", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956250-uZn2aZkONuuxxWI37G4b", "proof_sha256": "87e7961196a08b0eff27fa06e95a216af5d9cc6ea58871edeebc8b8991341efb" }, { "model": "mistralai/ministral-8b-2512", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956257-j9tgpFl70epyVunmN5Aa", "proof_sha256": "f4ce35e9bebd5428364a17711a9c906ca9ccb96d33758eb098f9af36199b90ee" }, { "model": "mistralai/ministral-8b-2512", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956267-CP9PIFohBz4ht1qG94ZF", "proof_sha256": "3f160ff49d4d2bda103726115e2ec42a69c5524a7b4faa1a4cb8f66a3dd49d18" }, { "model": "mistralai/ministral-8b-2512", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956277-2NdB5m7Ls7hthF5ezz7j", "proof_sha256": "d000dcb6c69ba4240b9bc4efec2921b87fee2bf3b012e1228987443d1831d300" }, { "model": "mistralai/ministral-8b-2512", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956296-OQPezGDYBBU8lyCfffa1", "proof_sha256": "82f333d759bce5a825ec8cc2d0a5b63869d03873c755c6d7469e183fe73f558f" }, { "model": "mistralai/ministral-8b-2512", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956304-cNEGd4mQmcPVCdvlgd9K", "proof_sha256": "e9ccd5cdfc3a8544e35433e765843a978de4a6a3bb6465cd3664d802faba20fe" }, { "model": "mistralai/ministral-8b-2512", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956321-mP5KX6mPVaRUqQ8Ck9Tr", "proof_sha256": "70e2445ae2cbec1c868b1553e17dca2af321200ef743e376bbd17364f90ab818" }, { "model": "mistralai/ministral-8b-2512", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956331-oCv6TthUoLCrv5wDBaht", "proof_sha256": "d64b6680a5e1cd5c72171cad6030d2758a31cd3430d24e54b93f0db1a8618778" }, { "model": "mistralai/ministral-8b-2512", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956338-omvL6IqiZmUjEJ3KpxMY", "proof_sha256": "d017d797d5abc8e2fb59943a0a7bdb1a2566c357c6c9e0323f7306b339e60306" }, { "model": "mistralai/ministral-8b-2512", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956350-IwpX62nc5u0FmS819kgM", "proof_sha256": "e2df1ed4aa7f9a6f926c0895c61e06ad9657cb1cca4ce99ef81f9e64421a81ef" }, { "model": "mistralai/ministral-8b-2512", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956358-Q6l7584v0bQEISYuRlTr", "proof_sha256": "dc59080896daa46e23a2b809c937030c2b6a6efb57543bbda2a3bf7ec4131a70" }, { "model": "mistralai/ministral-8b-2512", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956369-CTQ0tz9nrEGemFRcYlMC", "proof_sha256": "e7c5e15dcd83522f0416cfd9685d08106468e5a94b4964599e4ee687fe1776b8" }, { "model": "mistralai/ministral-8b-2512", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956392-Dt8NGHDt7HTnZquDaz89", "proof_sha256": "6e186b6b5698a06451b3a1942aebe60b657acbd56dcb65beccfc34aad8b74308" }, { "model": "mistralai/ministral-8b-2512", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956401-AnR6x2m6LBTJVhXZeGzx", "proof_sha256": "414b5a25d31a6ad2b0393134b09e2ac2d795782da017e476c50541a47a43128e" }, { "model": "mistralai/ministral-8b-2512", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956420-3RHOBYoTvXER0Begy96n", "proof_sha256": "dcad230ec828c37a1c827315f88cb8068d279acd128c76cc7c2a852d75efad9a" }, { "model": "mistralai/ministral-8b-2512", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956428-mY24GWZJJXlABgvwplqL", "proof_sha256": "8d875ccb5624b3eb64da6b6a842163ef167a772540e63ead45c54707d4beb74b" }, { "model": "mistralai/ministral-8b-2512", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956435-mnFvyVwk5ztoKXGh28mj", "proof_sha256": "3f00924a0354d430d9946f7d6f0a67025d63edcf42fb174790432b274f9ca08f" }, { "model": "mistralai/ministral-8b-2512", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956442-SaLmiqseiW08JfcqMpYz", "proof_sha256": "b9ab087ea49b439f2041d5c63fd41b8d3badf416329bfd87a396e57cf9b4617f" }, { "model": "mistralai/ministral-8b-2512", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956451-O6kFHkh6srlCgfv06C5O", "proof_sha256": "60c52ea8bd3010f776be46093ffd2d50b2115ceb7e1e5acbf20821f01f26a940" }, { "model": "mistralai/ministral-8b-2512", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "- sum: \\( (", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956458-oZz2BNU2kg781k2veosN", "proof_sha256": "c5bfaf575473e1015c1e6c47c81be2b27bb17670a3b48687af741cc0501dc177" }, { "model": "mistralai/ministral-8b-2512", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956537-5vxhyg1s9muJcgNrZ6hX", "proof_sha256": "2531356ecda36d3825d426fae587c7347774b437ac3f0ae38e320a207de603ac" }, { "model": "mistralai/ministral-8b-2512", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956547-Bb1jDDQ4hPGgja3q03Je", "proof_sha256": "df53dd5e26dd611d4a59a13221d4eb7293d0819d1508ae1f304a5f018f673929" }, { "model": "mistralai/ministral-8b-2512", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956562-VxINFv6v3ypAXtGh3GWj", "proof_sha256": "fe715715aef185689c5af5df13b7c5609ad6798e67c4900ea0561bdcf2ce48c2" }, { "model": "mistralai/ministral-8b-2512", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956574-QazFaVrC2A3MqLXRtYpg", "proof_sha256": "5d1f95d383a809a5824212253c813d6f24359eed948a3c5019a69641871ccbe0" }, { "model": "mistralai/ministral-8b-2512", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956582-3ONnK7Qkr63yy45KKDgf", "proof_sha256": "830f85b79fa7e2e071cf101ad69d24a19a9333faa9c154fb1a7bf64c13217098" }, { "model": "mistralai/ministral-8b-2512", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956596-QLx70EvmrnKYcowkhkNj", "proof_sha256": "6d8222d4c3f105303075f119fd490f1d7345ab3013b6c9175f3504826212f492" }, { "model": "mistralai/ministral-8b-2512", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956606-7D9OrXEVSdPY8r0vifAq", "proof_sha256": "776fc528fc87caea56867013868f2a0828ed6cdfa91989975af0ca03ac1ac32c" }, { "model": "mistralai/ministral-8b-2512", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956613-zxjOVRn6YfKMr5t3iSPU", "proof_sha256": "8549e9e1569f437867794485c6af4628fbe08a205bd569b51f734f5e9bce169c" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954256-78YTs2iIfls88DiLUrzL", "proof_sha256": "530f395b1d5ad8da1fc07cccf2e5eb37c3b320702e627f089adc68aae0f43d3e" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954269-pDf9HhZItWUdsm1VHPTS", "proof_sha256": "cf310c34717ef7b16182be57d11263faf1bc54ca07fcbf5f7f003d8088713e1c" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Io Net", "run_id": "20261025T200000Z", "generation_id": "gen-1792954284-BSsjqLYIWkmmrJPwMHbr", "proof_sha256": "e46ed79376d7cf19e39350e92842f114c19a6cac6470d0b34254609258b56485" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "we want to find an expression \\(e\\) such that", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954308-yz5aKeiCMeOBqrimfc8f", "proof_sha256": "0b978f3c95cd6c7f83570bfda70ac44a0458b890a6b589fe7d114b0e2c633ab9" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Io Net", "run_id": "20261025T200000Z", "generation_id": "gen-1792954323-5m4vjee4jzCQgwbpBoUW", "proof_sha256": "f4672d6eec6e90c094435d46abb64040c313a962361f97dde7500e7046307f62" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954346-HOM8udT5ZCCPDuLqtxeh", "proof_sha256": "7a098af53cd6ac4833f23a3cd020505b6b547d00a8057ad131a3e9e804b303c4" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Io Net", "run_id": "20261025T200000Z", "generation_id": "gen-1792954360-dw9N45fE5KY31JlvVN5R", "proof_sha256": "aef0f4e28f96e8a1374d558318bc401eb546d7cb8517b1921ba03443c41a4b35" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Io Net", "run_id": "20261025T200000Z", "generation_id": "gen-1792954374-g6TqESiorvoz7roOi4cv", "proof_sha256": "f0c7e6335c0f1e3870b5da7baa4140dfffbf1f1d6baea33ac05b58cde597bb93" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954397-gE4PQTPO0CrSyCYBSGNM", "proof_sha256": "d630a072481ebbd159dd59623aac663b9d00293c1ef7608676b74dbe418a7c30" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792954411-rcpNCosQoTAxCkDBLyiL", "proof_sha256": "df8f07a2ce9fed64b322dd09109025b48855c96025cd12f65f01cfefa4354819" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954429-ZSz440OstRqQgrRWEsTf", "proof_sha256": "178c2e3a217e044c3631a06830b51b49b78a95b474328409ef69d60a29faa10a" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "maybe the fraction of the particulate nutrients that sink each year is the fraction of the nutrient element input to the surface layer means that the sinking flux \\( s \\) is some fraction \\( \\alpha \\) of the total input to the surface. but what is \\( \\alpha \\)? perhaps its related to the concentrations? or maybe", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954441-c2FVYc9jUQ3h8J4a2991", "proof_sha256": "34284aee1a0f52849b56d005cb1d7d786d2f14f654d86e9a1e0301a9afcdded0" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954458-cUsERcJakClrQY0e3Te5", "proof_sha256": "283163b8c16af5e0a9f82cd85aabbb08a4b1b65b132048b3880202840d1a9888" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954467-3Mrba1E2NVx5DOc56FOK", "proof_sha256": "3c9792019fca4568b4c054ac8819e1317f841953220ecb0436c1540e2ba6b3c1" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Io Net", "run_id": "20261025T200000Z", "generation_id": "gen-1792954481-3CGdkU4Roc5f33M25HrJ", "proof_sha256": "53568c32b7abcd72948dbadc8688a1247aeb34683e141df1eddaa088cc9ee344" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954504-p5UeHk9jPysN5hSESn7y", "proof_sha256": "ebd05597035292e9322cf54d9ffa947c00dd065f5baa8705ac039ba426ee901a" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "maybe the expectation value {nlm} is defined as the expectation value of r^{-2} in the state, but with the radial wavefunction chil(r) as given in the radial equation? the radial equation has hl = -hbar^2/(2mu) d^2/dr^2 + v(r) + l(l+1)hbar^2/(2mu r^2", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954520-2JbwnVnSUiJc6dSv75xR", "proof_sha256": "5f8ba27f021f8cd8cf142db90c0f4ac39c7eab8ba4c3c7fe19f74d770dc38a91" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "but what are the units of \\(c\\)? if \\(\\lambdam\\) is in \\(\\omega^{-1} \\cdot \\mathrm{m}^2 \\cdot \\mathrm{mol}^{-", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954546-gmgNBgVzb19MgA4ZIGpN", "proof_sha256": "ed40accd19b82484027772c5199649799e2e9f348dba61774e7b3bac10b1988a" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Io Net", "run_id": "20261025T200000Z", "generation_id": "gen-1792954567-WK75hmh2CFQjvacf6Njg", "proof_sha256": "f664c8dab750f0c721f3c75848ab0fba402c32856a26c21ba039eae5d7b6f2fe" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954583-JPH3xILl4dMwmp8TRxWr", "proof_sha256": "80cc55580340ef5cefeca47a55a5d5efa0c153eb269dfa1394e771fa685c51a4" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Io Net", "run_id": "20261025T200000Z", "generation_id": "gen-1792954593-U9O7y4X9mjsNVk8jccAm", "proof_sha256": "ecfd2c2b24c54a7404eaf052d086c3df3199892712d6a44ada07eebba1897e6a" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792954616-5xyS9zsvwEmLvQsfexmw", "proof_sha256": "ccffe6ce43295fc22c05d80916fed3f5f642089be3138620d05f2f92f1c44ca0" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954634-EoSYypxPpaYChmHUB96e", "proof_sha256": "5d1bd3467b2b7527e4efc6b49775afb766a98ea833d1e6054506854cf26d863c" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954649-S83nXzPoHnXXSFfSNvDE", "proof_sha256": "cdc393a3907d65a99a51faec052ad41d76fbbf64783fd92d4b1654a1a158abae" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "- thermal conductivity \\(", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954664-u71HVc0Ypmc6cB6DWyIg", "proof_sha256": "db6cb6c34eb35e813b7698f7d6b0968cf06ad0ef07fa6977d59b1100ab7dd10f" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792954681-yKeK0siv91qXPbYBdSwO", "proof_sha256": "02e8f183c657e3dc9dea20df89070db965e18fce2f2b3997df3fb83b472648b0" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Io Net", "run_id": "20261025T200000Z", "generation_id": "gen-1792954698-4sDAc3EqY0GbakyLAwPR", "proof_sha256": "61806a8e9da36b69ba58047363cf4225b6ba2c330a58e92d0f6d7dfa3a772aa3" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "perhaps the medium is a uniaxial conductor with conductivities σ1 and σt1, and permeability μt1. the wave is incident from free space (ε0, μ0). the angle of incidence is not given, but maybe the result is independent of it because the", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954724-FmDFgdNg8wP6AxFBTHHB", "proof_sha256": "7184c0b5224ac8c3e2cb6ffb609e13af9aa4a8f44f446a28402c621899835cc3" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "maybe the cylinder is not exposed to atmosphere on the other side of the piston? maybe the piston is massless and the gas is the only thing inside, and the spring is inside? an adiabatic cylinder has 8 grams of helium. a light smooth adiabatic piston is connected to a light spring of force constant 300 n/m. the other end of the spring is connected with a block of mass 10 kg kept on a rough horizontal surface... it doesnt mention atmospheric pressure acting on the piston. but it gives \\(p{\\text{atm}} = 10^5\\). maybe the cylinder is open to atmosphere? or maybe the piston is exposed to atmosphere, but the initial state has the spring relaxed and the gas at 400 k, and we need to find the initial pressure from the ideal gas law using the given mass and temperature, and the volume is determined by the cylinder? but we dont know the initial volume. we need another condition. perhaps the initial pressure is not atmospheric; maybe the piston is massless and the gas pressure is initially something else, and the spring is relaxed, so the gas pressure is balanced by something else? or maybe the cylinder is closed and the piston is massless, and the gas pressure initially is such that the spring is relaxed, but there is no atmosphere on the other side?", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954745-a09bTJcfdHlMiAyuMnvo", "proof_sha256": "190a970ff2e217b7e58d5d015a8e3076246ef3bbdb91e4d7773291af0eb9d3ea" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792954764-WYfBe1na8VxWQ27NUQ65", "proof_sha256": "432f27c927d0f6a389a8efba33be2cd64df32df325fb833705cba5c0f4ac9a5f" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792954781-IZ2kv2kUyRdXv6BzmozQ", "proof_sha256": "a02968d9c55ef65d7e3991fe244e65f0cc69c7173f436aff9236a44a4919d5b3" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Io Net", "run_id": "20261025T200000Z", "generation_id": "gen-1792954797-L4qiNsVP49cjepOrgcI3", "proof_sha256": "fa7298fcae92d8ab6a8c44b034e562c6aa80eb1d2ec95e068234bea5728130aa" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792954821-kfE2uPSofz0rs1RqlITi", "proof_sha256": "2cfd5eca01f4feb6ca307f8776d5f20994892f71d10be00f4d3c2319a9f0b38a" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954838-nnvDu8wl4zYbvGuf7axD", "proof_sha256": "927bca5523351e679b7028f2999398b696c2d86640fb02099a49cfa4ef1de939" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "lets write the equations without assuming signs by using a consistent physical setup. typically, in such problems, the rod starts at rest, the capacitor is connected, and the rod accelerates due to the magnetic force. the induced emf opposes the motion, and the current decreases as the rod speeds up. eventually, the acceleration may become zero when the induced emf balances the capacitor voltage, and the rod moves at constant speed? but wait, if \\(i=0\\) and \\(v\\) is constant, then the circuit equation \\(q/c \\pm b l v = 0\\) would imply \\(q\\) is constant, but if \\(v\\) is constant and \\(i=0\\), then \\(\\dot{q}=0\\), so \\(q\\) is constant. but then the capacitor voltage is constant, and the induced emf is constant, balancing it. however, if the rod moves at constant speed, the flux is changing linearly, so the induced emf is constant. the capacitor voltage is constant because \\(q\\) is constant. so the circuit is in a steady state: constant current zero, constant voltage across capacitor, constant induced emf. but is this possible? if the rod moves at constant speed, the area increases linearly, flux changes linearly, induced emf is constant. the capacitor holds a constant charge, so its voltage is constant. the rod resistance has no current, so no voltage drop. so the loop equation is satisfied: \\(q/c + b l v = 0\\) (or minus). but wait, if the rod moves at constant speed, the current is zero, but the capacitor was initially charged. how did the charge become constant? the initial charge \\(q0\\) would have to discharge somehow. if the rod moves and current flows, the capacitor discharges. eventually, the current might go to zero when the induced emf balances the capacitor voltage. at that moment, the rod has some speed \\(v\\) and some charge \\(q\\) on the capacitor. then if the rod continues at constant speed, the", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954861-M7efD7jWk3qKH0zmWygJ", "proof_sha256": "dfae4e2536e05c615f83fdce1cf4cf767e1565d1caab78a684b7e3d33d7c0f68" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954893-tr9FvTqITuinnxc0ivPi", "proof_sha256": "479c2433d0db696b1f1ff8700ad8e4eadeb2fba4e9354ab6d6f1efa58294293e" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954909-DkHKu2aOaNZQRNx5VroK", "proof_sha256": "3935fda8809f389d8c521177a9e5b552309bc21939d188e05f2707ab2dc9decf" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Io Net", "run_id": "20261025T200000Z", "generation_id": "gen-1792954930-K4alGW0w7gMAwEnziuz6", "proof_sha256": "06673a618a2a85e6f366e5011bedf7fd315c14be0dee2feebc69ed13cc80fd88" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Io Net", "run_id": "20261025T200000Z", "generation_id": "gen-1792954953-NDnEJoxxFR7DzhNad3LZ", "proof_sha256": "97114b5adc6e87db2b4b9104635a0ec3cd346386ead9d45b9f9cc66016e84986" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954977-parOsYvSZJg7uwmzHxXG", "proof_sha256": "9b6331b03bfd9fa1dc3fbcaf8cc7d60af0341d97fb6e1089c2508deeef044e83" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955000-TE4HymsyuZVcmMzVlsSE", "proof_sha256": "bc9b36e9a9f4ac84464823c518c05e25e5a0d8a4ce4fc2569d5c053153ff92ba" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955019-OjwHBTdcp6UDvop8O8nG", "proof_sha256": "4d9a727119e274c8f8c80c0a61a1c16821ff2d153bcc36f20834f5bb6cc66078" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792955036-gKGLEitvurDOLtOozIDh", "proof_sha256": "b903e42120b6a935a5dfa6a8048eb0869792888bfb3f424d9a7a701f938f1be9" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792955054-MgKI3uBXQxF6Ay1a1B2r", "proof_sha256": "47d1a8c899c465ec886f3507b87101faf22d8c5d929f1504bd3c56ce034a9920" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792955080-NFL3BEHDOeNaHMevpdqr", "proof_sha256": "1a9d6ecc70fc5e0d7588dfccb4ec3e32b25d54d61b7ae7096a592bbcfae86adf" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Io Net", "run_id": "20261025T200000Z", "generation_id": "gen-1792955102-NzTVxdIJAtfYoBBUoFHE", "proof_sha256": "723b1d93e303466bd92fa282b693ce861eaefda69c1e3d2f19106faa8b7374aa" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792955132-9VzzfR0oJQTUHUQ5omdO", "proof_sha256": "6d220a2cc213db5b65d39ce73a7579ed592484aa6e8e85a707b152604d005371" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792955152-0cNyPy1CtRuksg8ug8tq", "proof_sha256": "9dd67d3bac099e23349bbc300a2d4addba5815ddb6ff6edd34da7f2eb77bce72" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Io Net", "run_id": "20261025T200000Z", "generation_id": "gen-1792955180-WG4wRKpVcUz0osTYNh1q", "proof_sha256": "56702b6cfb4e9ae3191b1979804c40a0084a6f6f7d713400dd23e9a071fae71c" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792955205-Gze8xdj4wcaSuLoMotPl", "proof_sha256": "ffd6b8ad29d8f45259d87feb35869df500ff08c6bd780424c0b86afed002c819" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792955223-iViCbZ4etU4FBUK9VHzv", "proof_sha256": "5c383edfde7df3ae8b2325fa028a42da2ed74eeeaf5f33cce2705d447ccd8dd6" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Io Net", "run_id": "20261025T200000Z", "generation_id": "gen-1792955241-deojfsCaB3X8zPZVX65G", "proof_sha256": "959ce2cf34029be4b470376fd7a3449db6afe74d3e397dcd16299c0faf3e95a5" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792955258-wMLOYLQgsH10EaHVJYZ3", "proof_sha256": "4784d3cc793b64e97b7895f61bd2d352dc3f3970d9996a19e3ab994c091944aa" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792955287-SGh7wsH7ABz9LdTahWd9", "proof_sha256": "51da21dcc08c964c683ffa74eb6033a210efdd9706fc5e5d4aa6a244aea750e2" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Io Net", "run_id": "20261025T200000Z", "generation_id": "gen-1792955304-5OX4YJtCeALe72de1zTJ", "proof_sha256": "5b383f6393998da97652b494484a7ae6df32915c01b48000fdaae3f1f896c4f0" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792955329-i4qZKCt0YXaaJHYupfdK", "proof_sha256": "353facfd5ac84e1c1f60831e63278f4df437e717a8220ff3dbeb55e1453b2701" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792955347-MKoCUOVzSj8YkWFmbzZW", "proof_sha256": "84a73bc53c626fd412689b8fbe402d001002bc32648cb77c2280867eaedccb79" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Io Net", "run_id": "20261025T200000Z", "generation_id": "gen-1792955390-J1LRUFAeIP4fYS6Ep6aw", "proof_sha256": "5d6663f244fa3c904b2d146c5869e3394b4eaed5f023db8a016369d5743fa3db" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Io Net", "run_id": "20261025T200000Z", "generation_id": "gen-1792955431-ri1K7JIlmu6qzi5y1axu", "proof_sha256": "43480ad525805b289153252f1d0238af0d1c350f085a78c555df02532b778767" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Io Net", "run_id": "20261025T200000Z", "generation_id": "gen-1792955490-E5Pj0VJDikHbNaU0EIhD", "proof_sha256": "57231bb445ee6e0e8a55e16b8a5d243afc9afab18d707302af76c6cd3587f74c" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Io Net", "run_id": "20261025T200000Z", "generation_id": "gen-1792955524-PPSRCtOZcpMOIIhsw3GR", "proof_sha256": "1eb57fa21cebe039b9ae37117dd8d207d7dad565091fcd9166eaa3343e3c9802" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792955535-ycetodTmSr5bVi2jKzwg", "proof_sha256": "fbb4b25bf27426153c94653e315c45c314936288972dd76aad91257a9564b140" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792955575-T86JgtOzmWzsVVkRmudE", "proof_sha256": "396312694774f86e938c9e90cfba72704bc4c374e1b53d18a58eac3454d8e07a" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792955617-ad9d8qEHaBOXflaQrIBy", "proof_sha256": "7ea07c242ea50fc2edd6d133b34c5efe1f784c27258e9f5c98e7843e91ca0f1c" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792955636-v6eM2VuR0IfxCrTQqsSw", "proof_sha256": "ab4c13eb4b643e6494d820b95ecddc09038cd2ed5cb58d87869f442a639e2bd0" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792955653-Kctu4PzWQmfxR9LHWLWx", "proof_sha256": "5a0216cc917c69532a90d245d4df6641a17aaab0d50355a8ae3355ca6e572fe3" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792955671-90CvtjpuTsrl848WgW2x", "proof_sha256": "72e670feaf74ca841ab5436597c3ebbdcdff843a1f1a0f569c89f7ec363108c4" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792955711-B1P0lnzdh6shEEtmrbQN", "proof_sha256": "5bae863a7336533bd17115344509d984a2f85ee332da53e9f01a3eb5da07da32" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Io Net", "run_id": "20261025T200000Z", "generation_id": "gen-1792955797-Af0es7oLbvINJGM2jmxH", "proof_sha256": "a1a24701828591cf8145583bd86881b5f35b1db104fa71692a8bbbeb5eafe8a4" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Io Net", "run_id": "20261025T200000Z", "generation_id": "gen-1792955837-mAbFQN6HDBU2VHb18hAU", "proof_sha256": "807aca9a5eb50f8eed66d9816b3226b18981e2a815f006e3a09d3ca770b5d7a8" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792955878-fyt9EVyQmM4mFmHxBczl", "proof_sha256": "b0d9ca8ab614b19bf2175a060dbd3b2ca7b9be1877f643ce73bb04f20d2b5f18" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Io Net", "run_id": "20261025T200000Z", "generation_id": "gen-1792955919-6m5QGanDksVbddvzJ8Xu", "proof_sha256": "f9263e9b316ad78c70a8238e0056fbf3080ece8056ea66824a2bcc7ade6c054f" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Io Net", "run_id": "20261025T200000Z", "generation_id": "gen-1792955980-LKjEaeN5ohfCZrxyYxfC", "proof_sha256": "f084b848841f8172721126ce1b5792c4eb789d5ee130b2a2122e71874db05e16" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792955991-RkBTZL3Fs5FvNdnN0YDu", "proof_sha256": "e52c4d9a1b16bdf4bc726eb5bfb9db39284154f08ca4ff587f586264c110b1af" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Io Net", "run_id": "20261025T200000Z", "generation_id": "gen-1792956010-uFAoiiQ3z1vXmYBYXFsM", "proof_sha256": "2ea27f2c791a9798c6d0b79651e430619296bc881d2b543a4ac078329819f4c9" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Io Net", "run_id": "20261025T200000Z", "generation_id": "gen-1792956035-Je6X1lWreZPGMq3r92OS", "proof_sha256": "19bef92f1ada961a1be0bcdfcb0581b6c36ce5d576efbd3b7982976d0e5225dd" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Io Net", "run_id": "20261025T200000Z", "generation_id": "gen-1792956049-9KIC7SaMYY9yjF4uRutw", "proof_sha256": "0fcaebfa0a968e3764e3a4f90ba4845b208b405e291cd274b539f6b6336a0ea1" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Io Net", "run_id": "20261025T200000Z", "generation_id": "gen-1792956072-ozoNBy3WxmskASURdnI8", "proof_sha256": "85d17bb4f3f88c2cdff4fd62b887f5a8f115e5deae0999e6d0345d7fc96d99e5" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Io Net", "run_id": "20261025T200000Z", "generation_id": "gen-1792956083-YwDiluS3vK71MuTBbuDm", "proof_sha256": "475b0e6eaa36ba840637813165d6f3294f2a68f70f24741377e8c25f49b1c5fd" }, { "model": "nvidia/nemotron-3.5-lightning", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792956095-bQQgO8RMda7PdHRUbQWd", "proof_sha256": "a4a859acad7c59629f1baac63e034f1177a1068d4b275be04dfa2f67f2883459" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954256-0j64ZpWPzuWmyiA1J2qg", "proof_sha256": "89a556761a12ad7fecd3476a6eb8827adc4fbcb3323a2305fdfd6b9dcbbeaef5" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954277-R6folhlPVg1hZQxCk7NV", "proof_sha256": "b2ee60bd7f188161324e7842d98c2ed107ecb775a4a325049f890e7dbe9958e7" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954290-LtU3x0bh04WzgxffzuPJ", "proof_sha256": "b4701c122033962fc453658d887f0548123769fa1d8dbe4cfa3e676555b34f0e" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954302-q8bV52SzE0wpuG9tacuw", "proof_sha256": "b86136306424c26f1f7cdf0c28c642754984e7547ff96dba676a4e26fd2aabd2" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954310-dB5BIgWumvHWLYaZXgS3", "proof_sha256": "875d7b56898183b33854446650d6fb14ada1eb56c048ae30feb0628ffe3048c1" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954322-GZjKnI8BUulJZOeExENO", "proof_sha256": "f36cb9aedbf877a164a3982e0733270971f36758a62e87398a3e8b0ed807c1d9" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954394-NHQ3w0OJHRzi9PX0Ib9q", "proof_sha256": "9774b268ea85184a1d629ddcc1c8d47988c6c69723f6cdea6ce58ccae0b74fe2" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954409-pFiz0XASkGJ3pZsI7MD4", "proof_sha256": "630cb4f2c197a7a867f9411a0e5568712a38b03c64a5ad2ae38b6650ab406ee6" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954427-VbVRwaKW2RJYWd4GNHQl", "proof_sha256": "5f0d884ee60da583987c13ba298c8d8b17a1b5ddf7b386010d39d8445c887e9b" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954435-wWSigNrEGq86LPtAl7GQ", "proof_sha256": "8e7617ebbb89691b80cbf0f489dee6cf07c1ff07f446900a5c10e21123d03db8" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954454-JpJwH1Cs5IXmTVPtwQHx", "proof_sha256": "8e8d91f003236d4ee9d5b8b0200f2adc86780a743d04afd920fadc2cff73cdd0" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954464-CSSv68p8AN1dRDzV2pMg", "proof_sha256": "2dac08c35d50573612a5d6002ce30468cfed76dd2251f5e6e94248cedc6d305e" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954472-2B81XZC7QDwV5QOuJSnX", "proof_sha256": "6fc057b77c2f066dbe3c3891a334bfe0326b05469b51a04836957d2c56e7b988" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954482-wfu5I5ZDY4Egett0FaXR", "proof_sha256": "f92ac03686f32452c454b29ed248483e1c7694553fbe40982123c9667909e152" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954490-20yL1Ers80eE92JT3nVR", "proof_sha256": "10969f585526ac218b4f60637f0f759b53ce435e010979f5e646efb87bdb03c5" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954518-qzdmuQZFXDG57aZPmcy6", "proof_sha256": "0e83614d72223e744b8dbc65ddcaad514bb4b0eb7bc026fd248c265c8b50280f" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954533-sM1rF49WQdkGcmTlyYJj", "proof_sha256": "791cb8dd8f3e5bfa55fda8c220d78dacd909ec2426a464d9eaa7baddc105106d" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954544-UtrxXsdbQHF6oaRhbElH", "proof_sha256": "920c583f1e0f839140d51c4983e49ed1dc8047d60514a47012e9624fb8123226" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954566-b1vhTs6bI05rWy23FuMP", "proof_sha256": "3c235ef7e9eae479d62050144191e915e2eb4385e44c7590e1f74330b5778bb9" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954589-YnfIbZgjD2Pysfi1cYGq", "proof_sha256": "dc7b356f05aad1bf6c04eb9e7c2d0805b25c02a214cded8364b1aa30cb4d3587" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954601-xc1pP3gNMZPzK1FKZfwL", "proof_sha256": "8b79cd83ad23ea962ee66b580a9c87c0698b8aceeede6165fb88166fce338a55" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954616-9hfSi8lCh8NVS5usnLMI", "proof_sha256": "9720f64880ea6c40e6a63d60a05aa8767e093f91668c553322459e8bb4f7b197" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954623-RzjTRvOt3eBQQUdNUwAt", "proof_sha256": "90b92c724af3713c65f5c2952c56c28167cd360717158631c9c61aa58102cec8" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954631-okrsfXQBgMKLMnRlziuV", "proof_sha256": "a19b60f888f9e3c7279893ffaa9af84c38adefcfc164dd09e746611049bf498f" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954639-rfVRiYXgVOdCOEKfOCu4", "proof_sha256": "5dd97342897ed1706b0174b7cd16283fbaa875331929deffd29af74e1c410119" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954651-DzJ3O0nQHqvVJ1VXVWNF", "proof_sha256": "3d74e12c29025012e26a9f532fe10e4d978f6147a6d653fdc6d720f3144d0ce1" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954662-7lIngRqwfwv5m5pZPoaq", "proof_sha256": "b0003d68d32d0733168ade3f30bfe7a29f4e40fe9c939bc86fb404d0506ea9c7" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954682-Htlid4DTWPTsL4tvmx25", "proof_sha256": "e9484e31797e843989a468174c2c7394b2f0ef746a58a11124b04835b278ab56" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954690-vQOLFs91Hd4l5t5M3sxG", "proof_sha256": "3b86a6b5f75e148c9449267cc18a0469b0008de11b0e319d2a758d48c6dcd3f7" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954717-MraHenpUTTqw0xLN3FSQ", "proof_sha256": "3d368e2312012f56a1cfc93b6dd51d56c60c0bc2fa53784800ef95da6d68cd74" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954737-1WKXLRYrvRu7LcngxaLO", "proof_sha256": "2e72b081b16c1a969f132a4931839620aadf5017720eb10cd5bd05c83cff5644" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954758-Sc4G8lCCzmQArpOrt4Fa", "proof_sha256": "c8816ffb14e132b7e071232cad1dbcd1f9ad944c870e212465fa515dfa4fe95d" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954766-ZkMM2K95uM9pFRh22Xyn", "proof_sha256": "a96e81a7347de84e2861b85cb85944dc5b4b42bcf89bdac02fc5534d82aeed39" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954793-u9EHRJ0HnHPZ2rupryZi", "proof_sha256": "f2ddc4843a1b5fabaffc16b9d1bd274266f957aa7548c4d37894b2db8e965c2d" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954801-ZgAawenJLZNcjTlw1olD", "proof_sha256": "4f2ca663c3cf3854cf2c6ba5fe58bf44bfe0014d3d78a8a33b044754b3442f28" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954809-fmLzQS2pPVjgETZ6F9wA", "proof_sha256": "92cc0e98281340b85b1c78fecd17c102d5b37cc67482cfb00b307c97fb757745" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954831-WW4lk0vxcd8p5OB1Jo04", "proof_sha256": "68239b50514824b2b246925f7792edb5b54eb168bb0f60061bfdbac90ac28987" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954840-uXsjCbmKPlPmvnkNuBOI", "proof_sha256": "72142c7047c4cf6447a42d1d42dbc19b7adb9cbae198de72ff4c4e9fcf468846" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954849-a6RA5LJvgzQHednU2PdT", "proof_sha256": "2ba6fac0f69cc13402ad86b71dd8e8d88e800fdfa918fcb248881472b85769bc" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954857-bqaO0WMtF1Id8nN2fgws", "proof_sha256": "b0d955e446abbebc7d858cedad10f7d7d0baeb9ec7c4673a8a4cb5fc712fa2f1" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954867-ioyBp1Ll3G8JmYSUJaES", "proof_sha256": "6cb0f2d5d9c22a7e587a052de891b3cf793a6979da47cc438cd8d7e6010076a8" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954875-LxcMB2WpIR0XeH19ua13", "proof_sha256": "8d162aaa9da21e3338bc1ab35413b87fe6d29d65ba4a5a62439f333d9a86f31f" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954882-gdXhZ7Eav7t3GfZ30UPW", "proof_sha256": "71dec0a6f50c1f5665117ae1e370d04b26626911859df862dcb9e2bd77f46489" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954890-lebOdpbAAVOJFiAThpFa", "proof_sha256": "d7f608886204e3ac1326c4e25057948752297048750e6dca569062395ead732f" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954907-hOeNJkKh1eKEgAr7kpuT", "proof_sha256": "30823a698c9989d3f042af051a8d4e4235641288b880ed56b3a1abf23f9bd369" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954919-vYr2iUrZl9kbHvK7xSxD", "proof_sha256": "8ea780552518fd8d13e428f67279a45fb29702cb4bb642b677f47f2e856f95ec" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954929-PQjKf5ldqsY6w5mWviGZ", "proof_sha256": "dc69fe6ede4401258a79245c915442ccb1e6f6de7d1c920621500fe637a2fe81" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954938-fYXXrtkfO5wp5e97a7Kd", "proof_sha256": "179602c9449581437bc296be0a265c3725e6f64691471f1a38f3c53eca8e3a38" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954978-LnytHLW5mhtSdmYk42zM", "proof_sha256": "2029698bd3a995e80b12bdc387e562bac61d9683f86affd315f11094937e1b55" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954986-4AyJ2DmTGRRMASHgbOOE", "proof_sha256": "08f7ec6329d19cc43a712a6aa9ce4a1c83cda1e7a08d0521598beb57974c26e2" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955004-Ri5Awji9xdIkqbOvMXh6", "proof_sha256": "8837c8ade32c9279e4e2e44e7cbdbcefb33b3d92a6c36f1587273e74dc0c7bcf" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955012-8ens4dijFN8JNZAc0QK8", "proof_sha256": "3f2868324d04a43d72c893379fb98f9da141b0263b244bdff45620404e4bec18" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955020-f18zpEeg9gSwn3h40a3a", "proof_sha256": "63faefaf06aff6e6cc848fe25f60c3fafa71e1a7ca50d710062861823a07645a" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955028-ABkmPzWfI0uEOpArDS8m", "proof_sha256": "ace5f13be2942bed8c54d5b98ae570d3fcf149e238f2ed04c8259c0024a4bf78" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955036-jEPPBBobDJTAECXrMm4I", "proof_sha256": "4b6c006d8fea96c6c0f434942c6ec7e926cf2c93df96666a9031614a982c6931" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955043-bBaaElgKN8G1Gpyik0Vi", "proof_sha256": "230088b84aab645350b61b35b4845e532e7062f093ba001eb0bb51233551c9a6" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955060-dVbCXk0p73us6OT0xTew", "proof_sha256": "f1227de34004d203bf51843b019b5d6a26d570ce12f6d0cbca0b9f9ecf6d219f" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955068-O0BOP57f9eEHJDlvV8ns", "proof_sha256": "f1259ea3ddb75a22a81d02a2da22354294d927df27046aa731785e7d94160d66" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955075-P0zmCYASyMeaivCM8638", "proof_sha256": "03b8383d9fe146fc3d5786df9c9825242b83730b4ae614a3d765c0162d31433d" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955083-9oyX0hFFvUWEYd1Mkd8d", "proof_sha256": "8adcde821d06c5f966de6927fe0eac6b46c18d70357950b21eee3a20b3e80f72" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955090-y2PdsYGScom06r30DxS0", "proof_sha256": "0d3d5c31505221516bad70cdfd8bd300deabe323c4dbc33598c4cc1fd0dd11ae" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955098-NRBDujyfigj6iavHNbhO", "proof_sha256": "dfff3e677abf4ac6fd9e83119e17cd2b0f3f70953aa8788090c4195d1dd7b6bd" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955106-VD7xMmMtMNd9CDzdvIPg", "proof_sha256": "884bc888094d2095e67e0a992c3a3f61b2a71467bd1512db0708c56a4e6825e5" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955113-zs3nCKG5ukkcAAj67xlF", "proof_sha256": "0de65b3f2ad08824093c51951600e3dfd39d578e3f318a409e65a04962803f62" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955126-jsRC3czrOiwXfLqSSMz6", "proof_sha256": "a2a593308c9ff1425a836a7ae2a2c2a8617cba870c269ced95886ec5ca1f4bc6" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955143-3uWiuwjfjRpPm8Xc9rr6", "proof_sha256": "583c7e41cc1ab8ee1d9cde119c49fc4d40b0e8db90e2d946f7be1befe02978e2" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955151-tqzPsvGv2fojYUvbMpX8", "proof_sha256": "d9b4c9135b36c6382739b2f950b3fc997cd2cbf60a13c6d474f37ce5abc20049" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955161-WkCeeRI40nOrCjEx7Ei9", "proof_sha256": "b052eab64aa7bbfb8d07f1eac8bd1db8228d1ec758628dc8e8dcfa1ef466eb84" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955168-mEQUkhYntM44PmuliIiH", "proof_sha256": "0548a328fb05d7ad98ce3b56599e3f3d2ca5ea21cde1310dce0527bfe76dfcf7" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955176-wSADFiTNdalNpuwyCs5y", "proof_sha256": "29e7fec064ad3d569503a33688dcbc4591f28dce52326dd923310d3c1cc99cab" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955183-W3KzIPc0OHHp5Kv94koe", "proof_sha256": "6d0335523ac65a89cb6265cbeb0157c943d748dbddeb2db5047af305a1f43322" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955191-KMXRXf2pVRBb00bWRJHp", "proof_sha256": "67520b8c29822f669aefa0337c212ba8d5414c40cd72994c849cb3ee1e566bc6" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955235-dEt5tbqCJskBfe8XQ2vK", "proof_sha256": "9375385cbfcee36cb6ab84c4c12a6b5aa7b41f07c74f7c5dd3179ea236626e7a" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955246-YdCCHlwh93rGFZ3zY0IA", "proof_sha256": "3f73c15a63be36140b35fbfdf01193f06efec5ea4c0c751042eee171a1c17076" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955263-7CAV0DWM4BrvIsqjRaOp", "proof_sha256": "31f97c27d656cba16cf0e556229d7ac71f4bc08d497a6903cc51e66c251b02fe" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955273-vkPYElgSltuy6s8hUxWY", "proof_sha256": "88984b486ee4e9719be795e0d179a57686bdf561b1ff9b3dd63e0f9d06f76ca7" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955282-w8qDmsAihG08LmRARfBM", "proof_sha256": "49790833e7ddd3043fc6141b9485ca2dd31eaf28869e78fcd45a3fb47133465f" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955290-3ioUcdCp78dxfxVE0PIe", "proof_sha256": "9cc5adeb3ad7d33559bfc741888c845c8f4101e65bb199d6d47fe60ce8760c61" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955299-bc7HCC9pEDihtim7T2F8", "proof_sha256": "60157c99e919c7a3541a70dce80d9a2b648eb312466eec4712eb36a55e4481ee" }, { "model": "ibm-granite/granite-4.2-8b", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955307-jNMposeWBBzKNFhdEXVl", "proof_sha256": "9cc3d6f6b54608e7cb31d36d3d570e1ee9c90b672255a8f54e0025342f953f1e" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954256-BjmPR1b2x8EBb4ZveWcB", "proof_sha256": "8311cd22dd8ed83909a189847b5e2df41fd78e42f52de2276d22fbbff6ce9615" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954286-H4cqbwZrNxR0gOYKaTo8", "proof_sha256": "2bffd00c19944c938493a8a8c44cf0d0713e2e6ede556ab6ff47e7607886d5a3" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "answer: c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954312-kUgmjDT4tF3KSjar9jax", "proof_sha256": "015b8bed6f715392fa89bb5d9c1220116fd820a34010a8e4a6c7c4e660f08a2c" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "looking at the integrand \\(\\frac{\\sin^2(x) \\cos", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954331-kfzu393XGBNr7LDHYro2", "proof_sha256": "ff2282d91b0530dc4ea5cf57c1573ae7f652f8056449adca2d7c1fcb6e6ca3ed" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "this doesnt work", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954398-Dx2a3sxlDnbBDQVk9YgM", "proof_sha256": "a889350fcb997e5c331327b3ee320e51969d93866f395a6900c69b18dfb18a38" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "\\boxed{i}", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954465-pnqkbDDmyEspsSYDdxLv", "proof_sha256": "7f968a19d1a86d6e5b7c3ffffc4dd12ceb999df3d621b62f5251266194522c43" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954520-aRoCuvATeJFBoOHFSWIP", "proof_sha256": "8e0356d5a0f6233a4ed2dc1761fa2563d60f3425abc94071ba7c4776f2cd760c" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "answer: b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954549-i6NMDXW21AnklGqAJsb6", "proof_sha256": "2cfb42d9aeb56c29ddec0f66f7147b000adcc050f2491bb3e5f0d1dff55eeb31" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "answer: g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954572-dFeAHiztVD1NmtPYYNMo", "proof_sha256": "ed738e8b6886d148e8142a7a9801f6d0c182f915639ab8b8bd4c0184109d702d" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "###", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954591-uiFSwLVC2H5qqsCGRJST", "proof_sha256": "3f4d94c0d11a5e9b925cf7273ff32427047f638a696f09eb5711c29a328e38ad" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954654-5d8zs1gZr1Q0jkyuUI4e", "proof_sha256": "c31688d37d318f6531f3b7798699327456117b4fba84ab7f138a2c53260d6b76" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954696-3DZixuhmKuZtBMV8LFAx", "proof_sha256": "fc3515736e77e5f9fd88dcf6497b4f08bfea552bd59ccdd942cd73428ef78248" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954725-Vsa6HvSenb6VETDhKY6I", "proof_sha256": "6f7fa5ad1d3bebcc210d4b8698a11ba417cbaed3a2e66818bcb5f743ea7adf87" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954737-Vy9D59Ihdrn9kZG9PObZ", "proof_sha256": "ec1f121c3879f8fe57009f75a8588158948ce19a29bbde3d3ac42227b9c633a7" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "thus, 960 is indeed", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954767-IDCm20XLKoMcG0viYwF1", "proof_sha256": "4aba10d2b66cbaaba750641f2748d75ab9670ac22c02055369e7731b97d7a3c9" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954829-AOokT8tLUP6rvaROSZXc", "proof_sha256": "559ac241ad6c744c43d48624dcfbfabf7276bab5ff4b2f11d9b047fb9cb9d685" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "answer: b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954881-ObSc5D6J0jcQa9h4UcIL", "proof_sha256": "dc723e752b5b5cb806f60c4193d378cfe0315ed12c6fb2ce31717c6932bf5410" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "answer: c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954896-mEwri0PFyzWVNbsdOFHE", "proof_sha256": "6153d997a421f3d83949a780d29ef1539e39c78288731ea30c7e2749beb27923" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "\\boxed{e}", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954938-TmkFsRMu2W5a1JZyxyJ3", "proof_sha256": "1ef82722b493092d708ed492d3a957bec7c0dacef6b1f477d39ee279fa7c86a8" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "\\boxed{i}", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954978-doJtugMaYRbQxXltxva8", "proof_sha256": "947f4cf756766c245846f8fa463b7e0c6f5e2f9a9b62d7e7f3ba9dab988fd9db" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955003-iUrbdBMlMOmxje8uvD3k", "proof_sha256": "3ac851c92eda46cf0227868a4b3b6950cea305ce5ba57a915041da34d44c399c" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "1. \\( i = 0 \\): \\( \\binom{10 - 1}{5 - 1 + 0} \\binom{5 + 0 - 1}{0} = \\binom{9}{4} \\binom{4}{0} = 126 \\", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955039-kYUHteSQ75TsROID5jft", "proof_sha256": "aa5256b6665cdc43d0a91e9c8765499b154e12240700310df7396bbc837b6b24" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955097-Q27bZDI4yHmilKdYCQ6L", "proof_sha256": "1b395b8fd10ee9974e8731308db86d56139fe5bfd98586e7ab33eec7795abbfc" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955114-3hZAZRv5fyg5EuF9xNkI", "proof_sha256": "64fe12be11884383434fbee66755be4dff34a64343e28434b0557e009a27f961" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "answer: f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955167-1sxHT9GGpZOx4LRfjRij", "proof_sha256": "9cb71c9d77e7853823c2891d01e170fe2b9339a61193de4511e124b8e58f143f" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955188-2GENIJQXQ198UqnSXHQ5", "proof_sha256": "e39f5def9e29135f2831cc0324c7eaa426a0acb67ed927e94ba8bda6c9bb989e" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955209-F0eUgN5OFkLwqvcmoDzJ", "proof_sha256": "d7796427a0417df8e974111d432bb89170f4808fe81bc4d6587afa95ea13ef74" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "answer: b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955227-ERVVFrOkfDRyVANzgqsL", "proof_sha256": "2f3b6b18f848108346a1e7da1abec579c4d117b54929df4ebf48a90b81ae0fe4" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "### step 28: check if the problem expects the work done by the gas to be the work done on the spring plus atmospheric pressure", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955258-B94kB7qGhZ1qePzPF1Lb", "proof_sha256": "3225b58c9c697f9007bf995c3dda989290ebcfdc234774f198b152e877439282" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "answer: a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955308-RJNROhwnp8F6xhcccZtX", "proof_sha256": "35640ab11c4119fe877868f3249a7360cbc74196f5f399669e5484231549ffdf" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955339-yZGYahI7M0In4gg8k4qV", "proof_sha256": "5507fb1ebaaea6032da34a916c147abf469f5d06bfa17437e298f3fc5d831eba" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "but this is not accurate. the correct approach is to use the consolidation-induced shear", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955401-3JF2IPWoRJEQzJIfRHiN", "proof_sha256": "d4cd67afd07584b346d68a5756a8a8031a737f9ce280638da8ac8c359be3d214" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "answer: d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955503-dtrnOnz26LJz6Bty72us", "proof_sha256": "f044f59c2a945b6812298dfa6c599b99425293eca0bab807e0a5b66ac51a80e5" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955536-p4qxzRxOISiMX8LAfVBM", "proof_sha256": "9638a533c34d9412fdc2169817fe20f924b747af0a6647fa4e4d633242be387c" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "the correct answer is", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955584-gFdz6GXOfIUZxselsZhu", "proof_sha256": "5eff507874091d8033200579a39112260004a0a1700ebc05192cd6b80382f241" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "answer: j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955658-rQ9QIAGzLyZkRX4hhXcI", "proof_sha256": "ec7efa2f7e8a6d9237bfe70b3284b2e8fa564187e69281295e6586087f522dad" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "answer: a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955709-TTEfPwbFC9M1nYsonqhy", "proof_sha256": "61db1209da96cad7cb4dbae6128308b4803c696eda3dc74ea97fa03c2d1be480" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955723-O5mGbVcFIxa0WcPrczKI", "proof_sha256": "3302a0a92e957ccc37b918205e9eff26800e1a0f3c5202abdbf8a196fee94b63" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955797-3fcVUyc9wxopwLV1fhpv", "proof_sha256": "1395343227647f1a51b5f3db3b61edee5178192fb94886373c387bccbfae4290" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955843-h2eVzj4CuSrljbjftYnn", "proof_sha256": "ac90d95ae6c4b0122efef393a8b07c33635a1f5e1f979910136a198f5c3e7ef6" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955853-5ETfZdGsR8n2eyK87Cco", "proof_sha256": "c01d3c8cbba92755f9ebdc094606861f30b537cbd3bbe65789ef198b1b5a11a4" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955894-apnyamTFACGux9SdrIPJ", "proof_sha256": "9ba489efd5dcc836f9640c9ceba004366ba73c57df07ea97882894f3cec2e461" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955950-JdT3V49OsOGMhsaop7r8", "proof_sha256": "f33cfdd719f47206c35a0ded78f7649ca9da207d3e86b761238d630677b9eecd" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955992-cmJmxDXioqwqBRKgfQH7", "proof_sha256": "08e6af80013b17ebcd323ee4a879f520dad18178be0ea87f938f2187ef0fd63f" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956044-DAcEDMsaNHxNXNXYtD0O", "proof_sha256": "a497045ad07e01f4c730f97e8d6c7973924f36c47200d364a14310fec0f7b9f4" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "answer: d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956057-8dOsKK0KYaFsjd9w5RGj", "proof_sha256": "fe93905f789799d7689a8188636c11d99f208d76baad4e61c414e89bbbf98873" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956076-MqzoKPJTEOQbH3cQBwi4", "proof_sha256": "e49192a44ea5815a148b7fcbac2c2699fa98ab10329deda2e42dc1a5cf046543" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956086-FLOjNkKI7AJgBtyAuKuz", "proof_sha256": "bff826771b31b153e8484147021d3514b07b4e9f4bbc866b417d5bc086be9c93" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956147-twBma6RWFkUlHd6ObQZ8", "proof_sha256": "445dcf289e18cbf7df2219401cf537fe8a271ff817467fc39e8a7dcc69d547c4" }, { "model": "mistralai/ministral-14b-2512", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956162-GMI2XJxX5OVxR0IJmAgw", "proof_sha256": "c78421fa73075e3d0ef176811ea89ffc9781a07e233ca1002e71c4901deb0ee4" }, { "model": "mistralai/ministral-14b-2512", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956174-S63I4YQeUIOn7cbKJ6N8", "proof_sha256": "fc06e939ea7f5386c444700c42bea97676bfde96a6d3b44640c2af6a09115e20" }, { "model": "mistralai/ministral-14b-2512", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956188-nZuRO14P08ZSxMt7jAem", "proof_sha256": "5eb522a2688d95449a04654c68fb6f477af92987c553e36c5c71f73b9b4eadf3" }, { "model": "mistralai/ministral-14b-2512", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956204-dLeMsdE8ykv0qxdFyb1D", "proof_sha256": "7e146deacf0d561a256aca8e774248eee73aed6d93c06e5a01ef0bd8ccbe2fa2" }, { "model": "mistralai/ministral-14b-2512", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956216-puXOGJAZgCeAI8NsNwRp", "proof_sha256": "7490791d665024b00cf7d9d091c9b7778cc5b85d91e8be5158f20bc461a00e76" }, { "model": "mistralai/ministral-14b-2512", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956226-1V4STykyTAmDqApAMK2c", "proof_sha256": "a1c05f719ea0ecffda4bf18dae2269ea8eb727d60923d2ffcefc6c872866a2f4" }, { "model": "mistralai/ministral-14b-2512", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956235-6bWjlIF1yUu1DwmhfxH4", "proof_sha256": "14865c2df23c26536a02fa7d2f3935e46a4d4bc8058cef40f37abf370c065290" }, { "model": "mistralai/ministral-14b-2512", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956254-GUTIFWzdianKluc1nLKW", "proof_sha256": "89f6ad259daf72a2c3081cbf1142562911b5df27804cfb2839c84a2475c5b1b4" }, { "model": "mistralai/ministral-14b-2512", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956263-12adrbrcAi7FYPo5SJg3", "proof_sha256": "3d64d239a394bf959af3303062cd5bb11c2c2f18ebcba47c53a011354fc2c086" }, { "model": "mistralai/ministral-14b-2512", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956271-jNtViRGTafU9MhiD9tl9", "proof_sha256": "c1529cc704b278a2964466ba066fe4c1de45c6e1f48c9186f7973e256ddc59c3" }, { "model": "mistralai/ministral-14b-2512", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956282-en8sdzSqX1APuLUzYFnL", "proof_sha256": "677f47b664f36fc4ed6411aba52381114ae5c4737d81783b959d7fe0916913ca" }, { "model": "mistralai/ministral-14b-2512", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956290-8JuJR0DPaGky6aaPXnJv", "proof_sha256": "df43f3975089c4021434be5e0ab785c164b2e96f7ab16d7d39d88262afa1479c" }, { "model": "mistralai/ministral-14b-2512", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956301-n2vbXx1z4GekROY9EqMZ", "proof_sha256": "5ddef2452995ddaadd58bec86b056e6723263ccdceb3455a193ab795caa52a9a" }, { "model": "mistralai/ministral-14b-2512", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956309-Kraw1LRPYMzcoWeHMzX4", "proof_sha256": "50dcb9856d2b58bdc053a7b8a49a48c01c8d069ccdbe4eea2923506120bc050f" }, { "model": "mistralai/ministral-14b-2512", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956321-pxlEwsFxcW5ou1u5yiQk", "proof_sha256": "a5d0031bcf5a49e292563eb90d156796129ca03a34c55c8170c336af41239b28" }, { "model": "mistralai/ministral-14b-2512", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956332-Bf2EkDCi7VECeWYVzWGh", "proof_sha256": "90d852e66e59bccb0eb5db903aad52d1a95855f76c7573ab1295f2d0067d7f9f" }, { "model": "mistralai/ministral-14b-2512", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956350-SdMv2PoTAu92TdOH1TnD", "proof_sha256": "8b44ed5334bad3d8ad6734e9524ff30c2c2ec73bf2d0fdd958a0629153102f6a" }, { "model": "mistralai/ministral-14b-2512", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956358-VX03TvM27DfGVgj5P9v6", "proof_sha256": "cfed2fe2436702b83d6d35c9682894af9cffb1bd2218b4300513780596430152" }, { "model": "mistralai/ministral-14b-2512", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956368-Xz0sQmYSIqMLcngpRhwE", "proof_sha256": "706504717a536bd55460d20c6e91a8a298958d5a77470b4bf49e12fb49f10be8" }, { "model": "mistralai/ministral-14b-2512", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956392-P7zkEyopW8LSFXQ8Ids3", "proof_sha256": "ecfa1b0d5e3a96f52392a542feef9158896af64b8c2168ccb96a2101c37e0412" }, { "model": "mistralai/ministral-14b-2512", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956399-wFX5mIkf7BpQ5HgocJ5Q", "proof_sha256": "9dffb0179875568069036cf59145764c7f7dc28513aa82d8692a7cd20c73fad1" }, { "model": "mistralai/ministral-14b-2512", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956409-CaFOjs6UzD0ADN7I5eeo", "proof_sha256": "1ac811f3c316e8e12da083b698a1672f1be60275663f796b589dcf5f265d0616" }, { "model": "mistralai/ministral-14b-2512", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "\\boxed{i}", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956417-8jitpEEQJTNKFaseWrP1", "proof_sha256": "225a796fe0eab770de5e9dc4658e00e00b3a374ef79c328f1177a20b58aa33dc" }, { "model": "mistralai/ministral-14b-2512", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956497-jxy2vAEDNkYEAw1OWK8j", "proof_sha256": "1c7c7b6dab004b32c2c9f31dc18f7d1b312d3c376b33b997d7aee6a29083ebba" }, { "model": "mistralai/ministral-14b-2512", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956518-gEjS5vOyHzowm2U7jTnr", "proof_sha256": "21d224cb052f4258f82bd688ca7ce16e46b98a7bad7bbe7fdb3232b1a65068b8" }, { "model": "mistralai/ministral-14b-2512", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "answer: j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956534-4NptUb6Tdyj75GsKbhP3", "proof_sha256": "b92d085ca87e973bc6f4e421c8e037b77b362cd376cd46aee0272d47da51333a" }, { "model": "mistralai/ministral-14b-2512", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956548-MPG57cdYhxMCloSONuq2", "proof_sha256": "940ffec67ccb995ffdbd6c64cbc06e786aa22c19a34ccacfeacaa32f629f375d" }, { "model": "mistralai/ministral-14b-2512", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "answer: d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956559-L8uPR2NxgjALUajisCPh", "proof_sha256": "b7bc28ccb5d4bdf1dac55176e8e1a18a0bb5e1b5b326eea5d9768d4c46f903b5" }, { "model": "mistralai/ministral-14b-2512", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956574-3fIyppeaOPj4dq185sjx", "proof_sha256": "57181c0efb2ed02b6de3f3258a03bf5cca39de070cebdea76c41b7d49a63d7de" }, { "model": "mistralai/ministral-14b-2512", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956585-TdQUOtKCYDB4Ud5wrBsl", "proof_sha256": "225e53b9da6fed405a9787972b5c659e8ed7497a05799daf5f75b6ddea0707dd" }, { "model": "mistralai/ministral-14b-2512", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "answer: j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956604-z19DpSyabnOnZgQvP38V", "proof_sha256": "946da1e11b4bb38379ff32a4ff74610513655e9260660894299347ba0d19d273" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Xiaomi", "run_id": "20261025T200000Z", "generation_id": "gen-1792954256-mLrxxjX70rgFL75riAn2", "proof_sha256": "9a5007bfcd47b2c3b3ec1d6f85c11751841f8f88ab2dc4ed7af2d4ffd3ef314b" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Xiaomi", "run_id": "20261025T200000Z", "generation_id": "gen-1792954303-0D2KcwGzwXQWW35Bvheq", "proof_sha256": "f1de0c4aa41acde77458feeb65f6ac50d444319fa92db91eca634bbfca0969f2" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261025T200000Z", "generation_id": "gen-1792954402-HIavlL3ccm6HE8M0SLVp", "proof_sha256": "a560cd29f98cff98d544716dc4c60883a6f7478533e2678aa8db55578740d9c3" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Relace", "run_id": "20261025T200000Z", "generation_id": "gen-1792954420-0eAbsqhH1pyy8HSdmU8y", "proof_sha256": "a9b511d1893d89310c85a741b98ebfeeaefed1a76f94b50c6387d96cdf8e5a86" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Relace", "run_id": "20261025T200000Z", "generation_id": "gen-1792954440-gtP5Y42ava1YCvDRfgzz", "proof_sha256": "dae3648ed72c55192cc2bd0ee29f1602b7b17d10dbaa7e2e54a0044407af9116" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T200000Z", "generation_id": "gen-1792954490-IrFRcaHP2HpktpusD387", "proof_sha256": "a6fc7ca9d6efa58550940308c379b178a7e69d4090cde4b5cb0631d17dc74c35" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T200000Z", "generation_id": "gen-1792954522-rCyBqCSW6rWVEfqzulKO", "proof_sha256": "0a2bcc30262f7e6dc837cfd5ac3509425a04329b6eb6f06ccaf6c1493c0c26f9" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261025T200000Z", "generation_id": "gen-1792954535-ltyTNvq2lZlcMrZ2vW2O", "proof_sha256": "167d86f8d62d30b2447954219125608bb29e86de05f25996dc50a5c237d3bc73" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T200000Z", "generation_id": "gen-1792954544-I37Ta4kJGomgxOJIjU77", "proof_sha256": "f96393420c3f180e533eb92e4d646d17265a7e0b95cf9b7d7376914fc13e9dc5" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Relace", "run_id": "20261025T200000Z", "generation_id": "gen-1792954557-TJZrCLQi5KEeWRj2mFae", "proof_sha256": "68a577755d35e2aa080d5d1a57b153d86b5b8a7c8902cc58e47206b30ba2def8" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261025T200000Z", "generation_id": "gen-1792954579-F6uK7Ch15O9A2TprpMNG", "proof_sha256": "bfc8c370e715c940ccf98794e5e1cc07fa85ed5aa60b76225f66c28e60a340c0" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Cohere", "run_id": "20261025T200000Z", "generation_id": "gen-1792954589-3ln4XXRrPG03nGlzmnkU", "proof_sha256": "8e765ca5ac52703abd8e8e70bd33ec58763936b5e42121b14352a3455e0322c3" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Makora", "run_id": "20261025T200000Z", "generation_id": "gen-1792954608-gAn5mMhfqgXdLjzpavst", "proof_sha256": "1a2ef17fc4729a9a1f9c72476d7230d0afe994427004278c9f0a3d772450786b" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T200000Z", "generation_id": "gen-1792954618-BYQ1fgTPdSBCDdqj31Ww", "proof_sha256": "5476c20b970dff6579fba25105226788a231f092ea5939721237b37a86258fca" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261025T200000Z", "generation_id": "gen-1792954631-SNTjhafecDVXiiC1pE6A", "proof_sha256": "bc0812fd213df5135640928aab06186d40ff0a028010e5aff19370b25bc5a47e" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T200000Z", "generation_id": "gen-1792954645-DiCor02DPUnmFYk8ZDrM", "proof_sha256": "9ff881f42d70efb803e6e662dfe3d8593dde5f285768fd7e5e20798cbb3f5bd7" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T200000Z", "generation_id": "gen-1792954656-aIfuEtMBeXyl7FBjC9ya", "proof_sha256": "be5c02a2b2e73e1fdcb6b3a6972492ef9ac7dfe4a9a7b04a44a9ae821991d317" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Makora", "run_id": "20261025T200000Z", "generation_id": "gen-1792954670-puW9Sou8WhnLdfdNSPjv", "proof_sha256": "ba8d1b7b42f22dd479119111506232cef506ae712f1b839220344657bab2075c" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T200000Z", "generation_id": "gen-1792954688-1W4b0YcK2EwpcP1m3s2b", "proof_sha256": "aa8b169146e27a3bfbff0a6105f44faedfc2f25ff3808e646bf7f6feb8cd3805" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Makora", "run_id": "20261025T200000Z", "generation_id": "gen-1792954700-aj07oqpL1vg6gMrNZygd", "proof_sha256": "7d205e2eeaf1cadf7a8654622e1f6e9a2580772965ffe738278a4c214c8a91df" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Xiaomi", "run_id": "20261025T200000Z", "generation_id": "gen-1792954712-8aJOkHLWl0MycEF9MYHn", "proof_sha256": "6d8ba400996281dfac895676a66fa9ef6d373586b47accf42d375cb6df2e6f9d" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "$$r0 \\;(\\text{before } a1),\\; r1,\\; r2,\\; r3,\\; r4 \\;(\\text{between as", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Makora", "run_id": "20261025T200000Z", "generation_id": "gen-1792954727-pArtQwGwlnVtCu16gr2y", "proof_sha256": "c2d9aa34ec35d22a2ac3c6179eca0e46a1212105248270fa0e26f40cc8c67a8e" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Makora", "run_id": "20261025T200000Z", "generation_id": "gen-1792954754-zOKRgu2QQ6fKDYHQoLpY", "proof_sha256": "cef1f6353d270ee9c20b4d1efe4e50dfe45e09c02af70f7833080499ce8eda9f" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Xiaomi", "run_id": "20261025T200000Z", "generation_id": "gen-1792954787-UWEim8hvZHzaU9Fyd8Go", "proof_sha256": "44075644975101af86ef45c2f7538e948ea45b79b62a04af4be1e0f6b7152c8a" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Relace", "run_id": "20261025T200000Z", "generation_id": "gen-1792954811-R0LnYh6d2hu6QQOuiryO", "proof_sha256": "2cd47c064a9cc01f4f34ae4903befe2f6ddd9b6893f00c80fd902edc96bcd24a" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Relace", "run_id": "20261025T200000Z", "generation_id": "gen-1792954863-8LqlGrX4floSzDDwfRPh", "proof_sha256": "6fecde30da5781ff2abcfd7107fd8e48d3d8e53014fd0c97a3e22e82d2aa54f2" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T200000Z", "generation_id": "gen-1792954883-aGGGQ7FqlU2Jg0MSj9al", "proof_sha256": "f9773bdee78cd5e231cc4193be040650192d031143c9029a23ad2971b92c1301" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Relace", "run_id": "20261025T200000Z", "generation_id": "gen-1792954906-LGvo0rOSHTNRyMcMxJ83", "proof_sha256": "6a31880da08da94cc5c9ba0ac68fd845d429bb79e7022c1388ddcc43ee3584a3" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Xiaomi", "run_id": "20261025T200000Z", "generation_id": "gen-1792954922-a3RQMchGWnzO2csBYLQ8", "proof_sha256": "0a642b3ec79422cd3067c798ef1540d6bf44b9fbc18fb980a00c77e5300024ea" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Makora", "run_id": "20261025T200000Z", "generation_id": "gen-1792955007-iTz6KEy9BAf6ocuDQ5I6", "proof_sha256": "ceb87ca3b7f86187660437387e89ce2e6fdbf2e7e63e2dc7d3a65d26d39f022d" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Xiaomi", "run_id": "20261025T200000Z", "generation_id": "gen-1792955030-L794wi0f1O3ElHgC63ew", "proof_sha256": "2b0c62c067549b793d105b4b697bf104967bed221f0aa75b5b63d663408ddcda" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Relace", "run_id": "20261025T200000Z", "generation_id": "gen-1792955111-AoBi27hSF4J05YB7Lud3", "proof_sha256": "a0f2922d64d4a8d374d2e34f5503d1a2bdf09d00a41ccf406db2887c8e9ab8cb" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Xiaomi", "run_id": "20261025T200000Z", "generation_id": "gen-1792955144-KNWXPvrf8Cs2pJ5clnKf", "proof_sha256": "2bc9a6af2f60f288cb06c6cde3c7597d90b3a0b470b67a4a024e4fd53f1ca497" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Makora", "run_id": "20261025T200000Z", "generation_id": "gen-1792955236-8sbj9DFjGHc8yA34dkAN", "proof_sha256": "e90189fa9b2720d4b4fe583a4f0005b4eb19fc1420cbc90ee5f3651f7b0787a0" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Makora", "run_id": "20261025T200000Z", "generation_id": "gen-1792955276-LzsgP3bF1iz5Csc42Jdx", "proof_sha256": "5d3055318a3fde4648ca1b73ffc4c519998f22ec5c351fd3e4e6dde7f14e5f0f" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T200000Z", "generation_id": "gen-1792955294-u0U2yLaPeDzse7DEuHpl", "proof_sha256": "e29c5ee16544303d0af21aaf25984968a683d9bb8a4aef36c0646eac90717ac2" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "anser: a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Makora", "run_id": "20261025T200000Z", "generation_id": "gen-1792955338-dvpUGavuvS2u6XjPQAwU", "proof_sha256": "d6aebfbdb48cdb3ff56c2cce421d05e219dbc11ac33afcb3d8cd77b63b1e92e2" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Makora", "run_id": "20261025T200000Z", "generation_id": "gen-1792955391-nMItCmurxpfXR3YwP9qc", "proof_sha256": "b869896c3ae2ea3d23994b5dfad6764149593c02bdcb71b815a4a8723da21ed8" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Makora", "run_id": "20261025T200000Z", "generation_id": "gen-1792955440-IDMPFx0vVGGyNl5STeek", "proof_sha256": "e665cb619f50129d22de1536f205e7b3fcd55fa51d54430e505ebfe48451f4f3" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261025T200000Z", "generation_id": "gen-1792955525-8HAHnhlkQWtfVhe4y480", "proof_sha256": "31abda9dd43128fc459b6e8f2d790276a9308e913b6d2128e4072bb7ec9bee95" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Makora", "run_id": "20261025T200000Z", "generation_id": "gen-1792955574-jnFZ5UC3ZEMLdAeETMJD", "proof_sha256": "2c179dfdd86f5754ecfc269726e6932d52ce9f2096746a190a2be33f2eec3116" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261025T200000Z", "generation_id": "gen-1792955619-57o2rxZj6pyEMhmpAIzj", "proof_sha256": "67dec4ee0fac244c56ba4e362e0c1d90f2fb534a2a98dfe5ef411d582afa2b47" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Xiaomi", "run_id": "20261025T200000Z", "generation_id": "gen-1792955639-9g6Wolrz9WAcIC3rT1tw", "proof_sha256": "9dc710a7a03cdb00b423b37fae4e1d4862d4c63914ec6d4ccb29c7df153ac5fc" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Cohere", "run_id": "20261025T200000Z", "generation_id": "gen-1792955735-QnakHnIhgWj07mIJnEMy", "proof_sha256": "16c372d8292d36f96f92dc2cd04289b75c7fd7d9a2ff82352787cdd9a371e8bc" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261025T200000Z", "generation_id": "gen-1792955797-zQgNVgOWePhjnSNhH3Xf", "proof_sha256": "eea2b179f4b2a04348105b4d29fd1dcc968dd58c9f77a02aa47f411de894e010" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261025T200000Z", "generation_id": "gen-1792955805-x2ddGE34rG2SE9g7sglR", "proof_sha256": "119c0cbd5106c8ccf183e4ca1cd411ae9afdd7cc0065b0c04e527a7e8c2f1aa7" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Makora", "run_id": "20261025T200000Z", "generation_id": "gen-1792955846-QSUWHoS7O81DFNsZER6E", "proof_sha256": "fd2eab1c302732377bae3e649f03349085742d1ffc11b2d4664d02ee15e1eb92" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Cohere", "run_id": "20261025T200000Z", "generation_id": "gen-1792955888-vUpYmw871Sl6OP4YDtP5", "proof_sha256": "71e16b168cae4f1ce3c3873c732c3a1255d1c8e09b9e7e6868cc056ad714effd" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T200000Z", "generation_id": "gen-1792955932-UuO2F7WjuuSy96K9qIfj", "proof_sha256": "301c2cb07e07ac027c1d0acb2763b7cf25ae7587edc6b3b2f79dfba810dabab1" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261025T200000Z", "generation_id": "gen-1792955980-92X8oWmP4VeyscLAAdEJ", "proof_sha256": "e8287e81b4f67f3cd161ab7cd595d3838d8ad82a07ce18e71692a0e2ea83f15d" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Makora", "run_id": "20261025T200000Z", "generation_id": "gen-1792956005-ZHvrzdMadVyzpIdJicUZ", "proof_sha256": "4d1c3adacf6af2d82dabb66fc11f0e4ebed3e4f99b89e96abd1488fe32488741" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T200000Z", "generation_id": "gen-1792956033-ypC5XKONBfB5FaHzsyn3", "proof_sha256": "fb73724568f11eec396bfec0b972ccdf5ed5d5ac10ac871460fc232b6df87611" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Makora", "run_id": "20261025T200000Z", "generation_id": "gen-1792956051-DRn7Utazjke7p3hImhH9", "proof_sha256": "59602532b99da2e3a69b2664ce43af2898bd6d3a509a2fe6f76f96e7e110b910" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T200000Z", "generation_id": "gen-1792956064-Ke9AKdSYSzjgt7JwT733", "proof_sha256": "6178bd43e7df63c53091494e9e7eb21e5e20b295aeb0c28d008734c7392fa54f" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T200000Z", "generation_id": "gen-1792956079-FlK5rQVlb1eXADOloojp", "proof_sha256": "5ed0ac73df3f8ffa64d8f316ad84427c184cadcaa1bc666af939fe2ce5cabfb3" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Relace", "run_id": "20261025T200000Z", "generation_id": "gen-1792956097-GUWLF0cgeLKy4GRxKm1t", "proof_sha256": "88d3ffb32e5a93efd36dbe891ad4effbbcf2285b56b93e090b8fe9cb63ad3fc7" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Makora", "run_id": "20261025T200000Z", "generation_id": "gen-1792956114-Ho2xF6885lp7PGITThL1", "proof_sha256": "95dc1cc84aa64077f40f0e13b2b5b7711ede1054afb53fe8266be5fea99de4b7" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261025T200000Z", "generation_id": "gen-1792956134-WTvYnYcKC6BWqqjG7xvH", "proof_sha256": "b807d0b36f3a5ddcf7ec4fec6cc84fec3966800ffffee3fc699ac951e9dcd95b" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Xiaomi", "run_id": "20261025T200000Z", "generation_id": "gen-1792956143-8SqhTOIW3QQ5Qa77mQ4V", "proof_sha256": "910bc181ae10f1d97351d71129e8e757e5aa3000cc390dfbc03360577c26fe44" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Xiaomi", "run_id": "20261025T200000Z", "generation_id": "gen-1792956168-CaLkhIW6CyVlvxYPrdVB", "proof_sha256": "29ab7cd3a6112b579976c34dc8a3e2ec5439083d373cd673583848b1638147bc" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Makora", "run_id": "20261025T200000Z", "generation_id": "gen-1792956190-Wucvu5URigwVGfxlcSjd", "proof_sha256": "81e7d1f5d820e4e5804b83452a552372dc89cf9390b83157f24effc1f9db587b" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261025T200000Z", "generation_id": "gen-1792956223-YdsX15ifBho8ceMkTLqU", "proof_sha256": "3d60bb32d1ce3cd46e324e1fbe758f12bcbd444ed2c36992f0400c215cd622eb" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Makora", "run_id": "20261025T200000Z", "generation_id": "gen-1792956231-of1qkZnmu5N2bF8azYAA", "proof_sha256": "32967e837138e43435aaa7533d552353905d31d93fd11194c352d21ec4940d96" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261025T200000Z", "generation_id": "gen-1792956254-FQDcREwOhj3vKzp20Sem", "proof_sha256": "d4775a7f146d0aa33bd3bb25309a9a023508ca895ffc0b2b459b91721e7657f2" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Makora", "run_id": "20261025T200000Z", "generation_id": "gen-1792956262-YJZhg16cSs9nvjyzW9KW", "proof_sha256": "6a9808748b79a28bdd6f650492c79aa4fb24d19a79ed9881e578f206e52f2b56" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T200000Z", "generation_id": "gen-1792956271-M7oXEoefm0nD1aIu7cij", "proof_sha256": "45b8eca491ed33fff86f36919a4aebae567eafa72273c546ab33f9a0083a4411" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261025T200000Z", "generation_id": "gen-1792956280-0aj0b97N5ifX4LLvGphy", "proof_sha256": "e3888fa7717e0788de07dc2334d2b779bd91f8e410e574280bebf735abe7b9d6" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Xiaomi", "run_id": "20261025T200000Z", "generation_id": "gen-1792956287-UoKropBOj9pDqfKDflNZ", "proof_sha256": "6b1689eab7958454eb9a5a4870fd3cf4c3e3ccbd3ae037225bb6dd30de1a8995" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261025T200000Z", "generation_id": "gen-1792956317-YPWgMzATnqTK0mkHovSq", "proof_sha256": "48ba281f7a940684f224ed7a1bdc68e7f03eddf8400fa3fb87693f4be7597627" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Makora", "run_id": "20261025T200000Z", "generation_id": "gen-1792956327-no4PsMeZDhDM40Z8pSoU", "proof_sha256": "045fbe3fdb1244836a41872fb535977b456d37dc52f7a3f8a7dc68e6752ec3f2" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261025T200000Z", "generation_id": "gen-1792956342-rkPGckkmcduI4C5QKFGX", "proof_sha256": "13dccb59e0338882d721b590b48fc7c1fe574124ab2dcabbd5156ab87b7c3858" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Cohere", "run_id": "20261025T200000Z", "generation_id": "gen-1792956352-3EAeSTIoKmiEpepHqDgu", "proof_sha256": "9e75c704599edb3508ff48f0845c00c1a860445e03bf19e8bf21b0e51ebe3932" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261025T200000Z", "generation_id": "gen-1792956371-qeCuavGjE9xI3COXMXnG", "proof_sha256": "16016ab63c9fc43039b5211dc87200234a7901ba4b5386624ba201264b6d85ef" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261025T200000Z", "generation_id": "gen-1792956379-GJSHZq4kbgELk8VeIFYu", "proof_sha256": "6eb830f2aadba6665a84d52cb092e217d0eacc996cf92f0ef075c4622e1cf33c" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T200000Z", "generation_id": "gen-1792956388-MjJW8NNrl9UXYQTuiOjC", "proof_sha256": "bc6a71c5d6813ef987653148b5eef3fab8d1aa0e47fb264eae0f7610439050b8" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T200000Z", "generation_id": "gen-1792956399-77q1kS1mesOvaTGArhT1", "proof_sha256": "c03e5253e2fda549e7209a68df6de549de4ecefe726e9eca7a9b8dd693c339a9" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261025T200000Z", "generation_id": "gen-1792956421-AO7Sk6dQoIyansNSxyew", "proof_sha256": "0417bfcaa8d904e5749f5c769abda0e6e9daccdb7dcdd520cc983467c1899200" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T200000Z", "generation_id": "gen-1792956430-zqflVbPSkLBOjmxdnf14", "proof_sha256": "6bc465d79eaa819c213fee2fb274b278060d8cd6f97b22ca877e0fa3b256c1b6" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261025T200000Z", "generation_id": "gen-1792956442-Cmevhmz75Kb4APSnlLuC", "proof_sha256": "a7ccd41f731977f39ff2daf4812196bddef972c99b96bfa05a00938c48dc7e1a" }, { "model": "xiaomi/mimo-v2.6-flash", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Xiaomi", "run_id": "20261025T200000Z", "generation_id": "gen-1792956449-yzince0h8WUSGj4Dw98L", "proof_sha256": "a84102b270b29f04372900cd45e9e69430166fc9fa2174233e985ec53e354b14" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "the final answer is: $\\boxed{257}$", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954256-uDBPfDUk49yU1Z9Gy5ET", "proof_sha256": "95e3bd21c665946293521832ad76098364301b2c8b9cbaf8d46b7531900b400b" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954299-keSn5wVOCKlUORmHwPFa", "proof_sha256": "0ee1f3a93a8e83396512b2b067f4ef87654652abc92ec630b272a81ab62f1de6" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792954345-BqTMamCtRVfuUdowX5QZ", "proof_sha256": "2615137f1de9b424bc49381610abbdbd0f324a49dfb882488720a1414abaf16f" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "the final answer is: $\\boxed{a}$", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954373-P8kUs4ujuyostlvKzSjm", "proof_sha256": "8261aa93ae52a0bfbe528ef9c2293706d00a25f17aadd75f1bfda95e75d60101" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "the final answer is: $\\boxed{b}$", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792954424-MdZwjmkrkc8T2fPZvLB0", "proof_sha256": "be9c9dff151e240a98987e983b30e8aa34b79abfa4020ee2fe54f381bea8f2ee" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "the final answer is: $\\boxed{860}$", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954453-9qvz7mkD8N11ugINIDfo", "proof_sha256": "fedbbd2d95df8a5f489c1440b708f99d7c4c9f97b22a28d27ac198aa96af1b68" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792954526-KQGnsa4vtliBxb5feSqi", "proof_sha256": "ab42ebe7dcc5344d7eb19367c21cd2df7f2b1b6640d3067d4b935f92959cb9ac" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954562-aZGsFqYVL71wtNlQkGEq", "proof_sha256": "bf2c307841f286cbf4a46ad31b91ec43aa847cb700a1df6b1aec365ad604ffcc" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261025T200000Z", "generation_id": "gen-1792954608-DM7bfZmYvUBT8ve91CRM", "proof_sha256": "485f582c69eb9021a549fdd0d50f71a1a898a0e87fb49edce5ab4fd9c52fa5d3" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "the final answer is: $\\boxed{c}$", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954654-P43vKIoDeeHEVvgisoT3", "proof_sha256": "1caf32b1462a383ee8cabf57a91f46d458ec25b1b74f5e20638f82f6e5a9f167" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954725-JFiJ6xm3s9aLywDNv7Dm", "proof_sha256": "ea2ac29024f61462f436eacdc5b31ebd6d78849a558313022340b12249c9ca59" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261025T200000Z", "generation_id": "gen-1792954848-gfBNz74JLvUSKFfqo74R", "proof_sha256": "1bec92cb611eb1d457ed24c0dbcd01be589c3195681ea5ea05c3ea00e13f8907" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954878-mASee2uqnJU1QyHvVuxP", "proof_sha256": "92fc8b73924b5c449691be3b0fc118c55871d5e73763872be9902838e1ca359a" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792954909-3sWOSBhekQzUDoMys7vb", "proof_sha256": "32d40e5c04af6486115579445b74d792c84878312a4e8c6d0960ee3fe2b17dd5" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "the final answer is: $\\boxed{i}$", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954943-HAL8iJeFHgY0ZABY5lNz", "proof_sha256": "218b3307d39ad61fae2c5c149589bcfd881f08548a181b92c0f4a3358332c2a2" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792955015-KMEjvMiny8lL4eZwRTIn", "proof_sha256": "4fad1277875a25822e9d30ce83feeeb5463b05583eb42ac5fc0d20f5b69bdcb5" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955045-x6TNe1zHqQRfQLtmxNhZ", "proof_sha256": "23d40f4588a91a5ba9a53673c280db31672f44798f6394339f45be8f33e0e750" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955138-WP8DWOdRqEwinQPOEwth", "proof_sha256": "ed29fb4f1267e9c63ffc2a276f38b5b4769734a26d41b11829bdfd07fb8cc582" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "the final answer is: $\\boxed{140452}$", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955241-BHCyxVIV28vA8eDw1HUV", "proof_sha256": "b5490bc4f4e58a622ef1f033ae66bb518dcb0eecb4cffc29e271ba22d974b0fa" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "the final answer is: $\\boxed{d}$", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261025T200000Z", "generation_id": "gen-1792955299-rYdjf4ewJcIT0EWKw2tN", "proof_sha256": "4ed2c5550ddd7460e28b7fa75c6c9edf6a5990d739c851a1dc7d0483ff67f107" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792955316-0tROLCDklwJXue576PMY", "proof_sha256": "b7c5bd69c617bae7984883163a4fcc842f07868a519078bceed87dc90ba8b4fb" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "the final answer is: $\\boxed{294}$", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955347-Mw77CWlbBYM8viiJxsBq", "proof_sha256": "bfbc52a08e32653100a841b7a03367de0f6b0afede7734e927d7cef003b9cfc6" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792955407-OnpWVwJYU4zS6BqlKIK7", "proof_sha256": "132f244b6ff9c3aabe4a1fb6c3fa1d58817bd199694b43698d2adef57c48ab1e" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955468-14uAs3EZQDbWMovzJxj6", "proof_sha256": "9e51409782eba944f84d4d198ebe0fd5038ff4a7b0c9009eb78be4b0a4d21049" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955548-yv0sP6ti0S44zMk0gT1M", "proof_sha256": "0be31b30a8a74c46e40912a4aa7f2d8aa36e97bcd647cad0798d2a62b2ae4c33" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261025T200000Z", "generation_id": "gen-1792955632-j9Tj5jTPcHO2X4hiY6Uz", "proof_sha256": "5e98032e05a1e8697cf7cff5ddcdf99cd3863a9cc3734b66596809adf1e845ec" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "this result still does not match the provided options, indicating a critical error in the", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955797-ZFU9B0p6jra4ttN80aQD", "proof_sha256": "ca3f6a0d8cca6ab1ef225b553348b7ae098382c13bdd5907bc7534ea4b469cbf" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261025T200000Z", "generation_id": "gen-1792955994-xjCFv1ueeTxYUW7YwuNy", "proof_sha256": "86c1a9bfd1324e477afe78ef83017162f82ef7022890e43a596e386de5854eca" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792956029-DRs7zE4ivHozH9YjjOqG", "proof_sha256": "8534c42f5d9d49cb4a63122f187e6e785a1c34043f8da6a9a33ff700e4577cc4" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261025T200000Z", "generation_id": "gen-1792956106-yJqwwS4suw79Em4MstJT", "proof_sha256": "94f2e475cb343821bff04e071a6180cfd70f044647ef57ba3760b5d798ae5051" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "given that the dynamic viscosity $\\mu$ is $1.13974 \\", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956153-RXxqqSOwLJtm0zpxPFDD", "proof_sha256": "74609b12028d2a506d7a75df880b6f07ef5defddd24696872085eacd5dd48146" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792956221-151Q8GbwFIY4qqSlZqNA", "proof_sha256": "981c0ac93cc77c188f7fd7f36aee6501b78c41c19e8f7542f34160e265570e32" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792956271-VuoxAwcsYUtJsOlVC5se", "proof_sha256": "3b4e5ef14290c35ffff91a851e7ce0cd1c0fabdd43466e9ccaac0da1b83aea78" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792956298-qLnTPj9IAzmmOQaefz3S", "proof_sha256": "5edfe186e37a6db329c7e976b12feeba22d47a191471165a50d3fcae207c57f1" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792956335-U6r8y8PTMPBP4qdbfmb2", "proof_sha256": "136478b465ec640e801647f67d4d581dbd33fbeb51d65ab65bc71a9715a929ee" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792956379-VtsSnIFB5O4QB2ML1MMl", "proof_sha256": "ceea7afb406c50d11590208a10d54e45f69675d0209044357ab62772db968fbc" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792956469-wiIuiRXKufNEbFgOKW2P", "proof_sha256": "3ac55a17305430af601d58688da8596f874864ae3139f3e4c55f0d39067f287c" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792956512-dbhbIy5RqfGajH20yTPB", "proof_sha256": "0dd12ce60ea2615f340099ef3fb05f532edafc453e6cd84358747467d4228ca1" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792956535-2FfoD5icunaRtFz13P33", "proof_sha256": "d0c1c5acb5def98447f77f320b1ced8faa67d93f0856d9ccb83bae0820c513a6" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792956547-LzyESN6tDLIedq2jz9AY", "proof_sha256": "42bdd6d44acf81d4b2cdb4dbabdd120e2d6e4dc603d440628f956b917f1ff60e" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261025T200000Z", "generation_id": "gen-1792956593-AveNtnCUWEifHvkUX256", "proof_sha256": "ff3e1a12c545cf60e01ac7e8da3aeb88041094c5a216ef793e72e2c5b167efd7" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792956614-MhlEsskzsmKkzHRBYFqM", "proof_sha256": "af9496d5aad04d45f2c17996092881627cb6137084cf552ef31458fb0462d68f" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792956625-5CABWyfJHbHIGtyzNtK8", "proof_sha256": "518017fc08b13904a41a4095e4c88da8dbf4511e20ba59144307ffbb39ffb72c" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792956645-0EkOZ4JmZqD2VOcwoDD5", "proof_sha256": "dca5108481456b189225d2742958c4020715b84b53dca6ec99fab469da21c654" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792956796-raO1T52k1Vhcbv5R9Tfk", "proof_sha256": "04ad23994e9ea3e48733c9289b0b3832503d0904cfdf225a3f805c8d66d7cec7" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792956840-q0GgNe6MvdZQ9OLoqjKL", "proof_sha256": "fcf949216acc8b504e96b79dd778af3ac0457f5eaca6cd41d9c5cd9e4a3c1a59" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261025T200000Z", "generation_id": "gen-1792956894-iXaGPTCs2aRNK4ef6M3S", "proof_sha256": "aa0770f4d58acbc10dcfd638ec6e2d7ff65d771de2cdbc62287000a94fd97738" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956925-kENuVZE0qQI0kHJxOtEB", "proof_sha256": "93977d95a8be8abe5a6e0488d3a05644b96348c0d4fdcfcc812e410c027496af" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261025T200000Z", "generation_id": "gen-1792956972-pbulEvOy1iY7c7Bg3A6p", "proof_sha256": "7047370198ad901441ffba7f1838ef69b02966554c3042b5eaaab90ae13c251e" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792956990-XNT6FmrHYBCmPrgxMZlg", "proof_sha256": "3168b68e039603402357f2ec60528ecbe7ba9eeffe768484864861416c779311" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792957024-95sxqrz7rPSoKTA18epw", "proof_sha256": "c4ea12e90754275102a9497e5187c0f79791932fb8e0fb0be86ba830cec8891a" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792957049-pTpzazl6ta1QYjKfZ9VG", "proof_sha256": "7ddd96c30851297b943647327459ba054697112cc398ebdcf9a2986963af6056" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792957060-d0xecSPbqJymSdBTFso1", "proof_sha256": "ccabb6b2667f6ad2a3dc8316e2f4b19113abfd7262043cdd981189dab67bf8d5" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261025T200000Z", "generation_id": "gen-1792957076-rw19EY47lPE8NU7zOXkT", "proof_sha256": "afb39bc84444bf464baceef2e38248998a0038db94874fdaf20ef6db0c9cf4d4" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792957087-GHp89NamCWfKIrRKNdBs", "proof_sha256": "13ffc9afe77de71419604dfd2d7673db5e40bb18eb94336699fcc0c8ecd7bdad" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792957099-9jKxVL3ufDYNcQFN7BBZ", "proof_sha256": "c762e4c4c925830a3a5e940150c665d68c6737defec8fae8d7e119d965eb3260" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792957118-BH2M0hmLgcbBOUOYFCxv", "proof_sha256": "9d3b14958655a37f8e6b8e300af006a6819a09c4b7b59168ea8c0cc51ac87657" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261025T200000Z", "generation_id": "gen-1792957163-LCweA6QzWhmNitqjDXaa", "proof_sha256": "6af62a197c8950073efee8b00fbaf51f061bab840cd616c2f18b0fc63445321b" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792957179-jlngTFVdDyD0x8DYXVeX", "proof_sha256": "a4d352ab5e60ca721b7ab99196847beb0f0fd89119143bf5160576c53ce58a60" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792957206-f5Bj2mifduXIAw0F6y6a", "proof_sha256": "6175fb196d4a5373be85a0232d22b0703ec2a534789d366316a29345e5a1cb90" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792957228-h1uQtdz1HYFzf2sQupZI", "proof_sha256": "511226355ad8cd51e5c4fb55a79556fa9acbac0f43fe35e0b2c720831411079a" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792957259-zPQOe2xduNbAovfZZwK8", "proof_sha256": "44c0d6c61e71a063dbbff3e896f85822fbe9a13336bf356f3c54be865b00649f" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792957269-ztqZSMk9tdsq43L87Rym", "proof_sha256": "f33828193b3683e28ff3524e37d5a33c604354446ee8b9eff3e230f2c746d5ce" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792957284-H1RyDGWINhSZK1edAw3O", "proof_sha256": "e78d53abf5de8bc690946464ff5817104951b1860571b6754a45daa3be1137b0" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792957295-GCsKslcxLtbGPl8FNfcm", "proof_sha256": "531a8f530048583ea82050dc144c1c38fa0b5d8cd703df5740bc9f3d4ff283b4" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792957305-A6MeIvJQvSC656O7t6Ai", "proof_sha256": "af30be0d50151d292984527341460d699afff4fda50af243c42cea790f10aba9" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792957315-ybjqKJpf426h4lUOgzF3", "proof_sha256": "68d747bd26e4ee06db99ab26c783982da0515f5c632dfb93e2b0f436056cbfc2" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261025T200000Z", "generation_id": "gen-1792957339-oQAp9PIcgd9wh5ri2dyC", "proof_sha256": "a1607f16db9396c2dbe25b4841ef06177b9c63f134bfdccd4e7d43e028457426" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cloudflare", "run_id": "20261025T200000Z", "generation_id": "gen-1792957350-IkETqowbQ8niF6D0r0Qx", "proof_sha256": "0ee494bd350486cd69bc043abdc6d3414058ad9d40acf6ab7071cfffc8b8a820" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792957359-AgT4elFaf7vgnNTLrrPf", "proof_sha256": "8c515a18de4253d2ea58a67f41a92144c53d4515aaf13c18c82d4a559986b1d8" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261025T200000Z", "generation_id": "gen-1792957382-7rOrdU7QFqHpgNqKQ6ID", "proof_sha256": "38471ce804e04f8c6acd9b1d88ba894998b32b3db49c242efb6eaeed90a5675b" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "the final answer is: $\\boxed{3}$", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792957398-Es6u5fN1oWIV1LYdnBFc", "proof_sha256": "da0459f008bbbefd73d880b104781dc5acc24f6c1cf91aaeaf1b96f3c6ca9001" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Groq", "run_id": "20261025T200000Z", "generation_id": "gen-1792957439-cpAXhAjZdU6rbZ5yC8tW", "proof_sha256": "1a0ed862643a2aae85ee7d936047e309a0a3cc47808845a25d4e032f9ed485fb" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792957457-VPs1N5CAtdHRoOPF8Nj1", "proof_sha256": "350771aea5edde5255eed60ba58130892846c8964d2d8330c12cf761ea80d33d" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792957483-y5WsOLIzDqSXaR9y5004", "proof_sha256": "27ace6fc70f317a3cd946e903b6abcfe0f1c296abe27fba7f2601b4125be40a4" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792957505-e4EkCh86eoLnjBCoQU1n", "proof_sha256": "da9fdb371c7594e3bf4c1cd934363679c221b353acb9dcd6db1d48b74f6cc8bc" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792957519-RYtxdz0NvAjnKU2NDxTg", "proof_sha256": "467060aae6bb1242136c3bb5e7bd64047944faad7386785389ee7a2dab6f79a8" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792957556-dCvtPCDXZfUJjSUv9yGZ", "proof_sha256": "2a128aa42a48a918c954c863ff283a3063f52f484af9bb3f72c76eeec3abc425" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792957568-oeMkV6KDvb0JE0Zus8lF", "proof_sha256": "f5edba98c478e6003f3d820a5fd3cb9b58c36805359fbe722f36abbb00732160" }, { "model": "meta-llama/llama-3.3-70b-instruct", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261025T200000Z", "generation_id": "gen-1792957579-Q72StVaZxRpIAqFQZvzs", "proof_sha256": "7cd140a95a30f6ee3f67dc50b50a899efcf01e222393e7fac08f2e4b1da368f0" }, { "model": "tencent/hy3", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "GMICloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792954256-OIwAbv2BX32Otm6RvobC", "proof_sha256": "ac7505f01c52490e21fd01f4cab222921b5c626bb2dfe79eab3412aa02199760" }, { "model": "tencent/hy3", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Tencent", "run_id": "20261025T200000Z", "generation_id": "gen-1792954280-WSLnVrxc02U6swOXP77g", "proof_sha256": "2722206cefce8ab0e37b1312d1b8a77aea6792e4043c02e03b90f0cf696a2c7f" }, { "model": "tencent/hy3", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261025T200000Z", "generation_id": "gen-1792954326-jURT5MInMRlcUCA90cdA", "proof_sha256": "01be576bc133a1851ac49e9035ea8c100c52a7180b65fae89148113ec52ebfdc" }, { "model": "tencent/hy3", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "GMICloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792954362-z3k0LSR1f9HGdYHPjxww", "proof_sha256": "4a47e900959b3e7f9bbc84b3523716778c0f611cb889871690f7739094fce4dd" }, { "model": "tencent/hy3", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261025T200000Z", "generation_id": "gen-1792954399-UmiwaYa2GS2utWIJOkEv", "proof_sha256": "f811ef11b8e2b2230cc96478a7984ed73d050bd4373c118ec8c9c8c1a187d4ca" }, { "model": "tencent/hy3", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Phala", "run_id": "20261025T200000Z", "generation_id": "gen-1792954442-BJN9Ps0tM1sUvrDcqNiz", "proof_sha256": "74a737423f16a5efd0f640a9875c82d618750f9825f04e40b1adafe5367cec17" }, { "model": "tencent/hy3", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Tencent", "run_id": "20261025T200000Z", "generation_id": "gen-1792954476-664CGWyhmzW3KEOlnLZi", "proof_sha256": "8ed1a58bd002e33642d0f5889648fbb7ef376fd22093908fec2bab76cbe2d7b8" }, { "model": "tencent/hy3", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261025T200000Z", "generation_id": "gen-1792954510-aassOREf2hOLiXlLPRA6", "proof_sha256": "f887ac5c4857d974fc1f214ddce3b24d378f5097292db83310a21b9e59a5085c" }, { "model": "tencent/hy3", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Tencent", "run_id": "20261025T200000Z", "generation_id": "gen-1792954551-Sovm8ELWV1kgfS7esFA3", "proof_sha256": "ca76b387976fa386a9cd01d4fd436ba778ffdddbf9d5a0a7a06e3038aad9df9e" }, { "model": "tencent/hy3", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261025T200000Z", "generation_id": "gen-1792954586-2nClEd5f119Gqiem85ks", "proof_sha256": "740285e2c0261773de142970df005cf8e91c66cf64db253b0292caf60030d5c6" }, { "model": "tencent/hy3", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Phala", "run_id": "20261025T200000Z", "generation_id": "gen-1792954635-7qtOdX3BCYxCCZlM9T3l", "proof_sha256": "1380c2c63dba060e5327f3ca1d30daf1dcc987459e3bbc6d7da55d1c9a146c17" }, { "model": "tencent/hy3", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261025T200000Z", "generation_id": "gen-1792954651-jM5Qfc418WBAqQQzrYMO", "proof_sha256": "c34b1c2bd9836612fb9d66106c9ac9f060c0cedd18ff489512f01e419746a878" }, { "model": "tencent/hy3", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Phala", "run_id": "20261025T200000Z", "generation_id": "gen-1792954693-wkqfd9AR1OsVf6CXfQPX", "proof_sha256": "826dc9b697e239809a4d5245434f3730f7cb54e4e94398ac627ea37a4aac4759" }, { "model": "tencent/hy3", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Tencent", "run_id": "20261025T200000Z", "generation_id": "gen-1792954707-rbNaiPgqjTlzWv00GP7e", "proof_sha256": "6243cb63f623ec4b812457199f1a60cf587a6440770c97361a5fd0dac204dc94" }, { "model": "tencent/hy3", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "- if $k \\", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261025T200000Z", "generation_id": "gen-1792954733-LM9BX7j6YD5HeELpzfEN", "proof_sha256": "7b95379123ef577ed4776d3bab9919f579f7ad7a31626f932d74d61e576274b8" }, { "model": "tencent/hy3", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Phala", "run_id": "20261025T200000Z", "generation_id": "gen-1792954776-9P5aoItPSY8D6jFduFSt", "proof_sha256": "a9a28d94c95125a2fdf2e14cf6b5a099def606decf38cd7b3142f862b09943c4" }, { "model": "tencent/hy3", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "GMICloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792954819-lTkmOci5BpDKsGKdtBRH", "proof_sha256": "ab0bb58db7b05082a58a069f4a4a5555277ef99d85ab008bfb3e9a0ef9f379a1" }, { "model": "tencent/hy3", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Tencent", "run_id": "20261025T200000Z", "generation_id": "gen-1792954861-Qb9gQA3O2nQXHMjlrax0", "proof_sha256": "5f0cd111119368f8758fe0a844e66af2ecddd30a1c2f20ed9535c69a9a1e3335" }, { "model": "tencent/hy3", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Phala", "run_id": "20261025T200000Z", "generation_id": "gen-1792954899-Q91nf8Bn9u4azI76nOzx", "proof_sha256": "40b33aa4c7a08afe473ed665fea4d6c10a9803718291bae8ce4a1498174090d0" }, { "model": "tencent/hy3", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261025T200000Z", "generation_id": "gen-1792954918-nB5tiq1R1Nh1Mv6POpJI", "proof_sha256": "f0e338b7cc8a44267d1bedb271842fb11a2641f3b50d0e7ce1fcf8cff5de88f2" }, { "model": "tencent/hy3", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "GMICloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792954938-TdVpe8ju5AwChAjuMGmC", "proof_sha256": "bfbce06f0acfe52f9f697908890a9938248b2d1a8909205d9d6a0078e8ae3239" }, { "model": "tencent/hy3", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Tencent", "run_id": "20261025T200000Z", "generation_id": "gen-1792954955-b6qYpgoqM10gmo8N4xCs", "proof_sha256": "78acb5e2fc51970c9c2af1354f8e6f46e2adc78688ca574fc354db8c4113c870" }, { "model": "tencent/hy3", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Tencent", "run_id": "20261025T200000Z", "generation_id": "gen-1792954980-Y8nDapDC3J0JZi8ICw3T", "proof_sha256": "77e70965447cd07a0ba866df83ff2a1a4f5718ee3d9b93503cf939fdff99d624" }, { "model": "tencent/hy3", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Tencent", "run_id": "20261025T200000Z", "generation_id": "gen-1792955028-n9YPrb0g2pAq4eHl1k4R", "proof_sha256": "563fdd646b6452eb4fe6e4379535e2dad22dc6e3d8105205ab18c7ebd9613d22" }, { "model": "tencent/hy3", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261025T200000Z", "generation_id": "gen-1792955075-ShCeDM9pfhSYnCH0E0pC", "proof_sha256": "2f69b3ae5170745cd5ca3ecd4e9acf1db005f3e2d9dcf43c2e95ce08171c244d" }, { "model": "tencent/hy3", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Novita", "run_id": "20261025T200000Z", "generation_id": "gen-1792955126-77HhrTMWQWi18mxCYed9", "proof_sha256": "9cb5029a0dd00e70a3405020cafcf9ceb6a06cb052cae4256ea67a1a51da1ab3" }, { "model": "tencent/hy3", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "GMICloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792955171-Y1tzYYtiowy3XrufinfM", "proof_sha256": "93bf651b1195b7af315b8304e5d6b374104dadb553b4bf5f5bd85be72d6de48f" }, { "model": "tencent/hy3", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792955210-mTXJBO5wCMQ5QG9AMYCW", "proof_sha256": "1f6ea5c56427f02aee6ab9d8c0bb448b0743efeea95c87b1c6e91f60ed5db0f3" }, { "model": "tencent/hy3", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Novita", "run_id": "20261025T200000Z", "generation_id": "gen-1792955251-XUyFSAcKfE13bV2h4svH", "proof_sha256": "c7caae8e0d16593f2a8769f79eb248636a9419bcf015803b130758ea07fc30ae" }, { "model": "tencent/hy3", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "GMICloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792955302-1IGm91EvjBI4OrOTZt5g", "proof_sha256": "c704fcac97d5fc54fd5b17ae12ec600924bab8d010a623311a82f69407ff2fdc" }, { "model": "tencent/hy3", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Tencent", "run_id": "20261025T200000Z", "generation_id": "gen-1792955373-yWjfw8rBmCXLOL8ooVU6", "proof_sha256": "80d82fa2b5cd4bd83d703ddae6b87f16d71b11e883cbfa7781f4ce6900fd3b1c" }, { "model": "tencent/hy3", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Tencent", "run_id": "20261025T200000Z", "generation_id": "gen-1792955456-L9LwW9l9vTsxOrH8cWFe", "proof_sha256": "9872e0bc65288dc7a69a29d93f77db35ff85904ae0d56c0e88110e42f8bc3d26" }, { "model": "tencent/hy3", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Phala", "run_id": "20261025T200000Z", "generation_id": "gen-1792955543-EpOJplgGa6hFdFpr4lkt", "proof_sha256": "f4cc946ce6c49177d4c3bd637d323c43fbfea0e1559170b9697840f541b4cc8e" }, { "model": "tencent/hy3", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Tencent", "run_id": "20261025T200000Z", "generation_id": "gen-1792955607-z4QKm5S1VSnEjSJKik7L", "proof_sha256": "31f7279b6300be9be8db8201b8a3149d1771279f19c6b3825d4a21b1916fe916" }, { "model": "tencent/hy3", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Tencent", "run_id": "20261025T200000Z", "generation_id": "gen-1792955660-oEK5yJItBa0bZpLNqk77", "proof_sha256": "8f9a8f38f580e0f8f64024b6c53031549c525d775e6a89096af8ae7109ea0b83" }, { "model": "tencent/hy3", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Tencent", "run_id": "20261025T200000Z", "generation_id": "gen-1792955721-GkRjpcm5y6osD1WpsaEO", "proof_sha256": "49c0063dd4d962b565e23330bb930e1439147dc68e34535bdd21acd1f193eae6" }, { "model": "tencent/hy3", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Tencent", "run_id": "20261025T200000Z", "generation_id": "gen-1792955797-ItCtle5Py7DxWHErRUoq", "proof_sha256": "9625bbe2895ae28a76e27a6b6866f065649532ea3a832846026c50d1b45191b5" }, { "model": "tencent/hy3", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Tencent", "run_id": "20261025T200000Z", "generation_id": "gen-1792955875-MWoKbl5H9XPjta07mioM", "proof_sha256": "72d307442e7ea7c7c0ccc7037877240e7036104719ca873bf1e64b8d34b9bf3a" }, { "model": "tencent/hy3", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Tencent", "run_id": "20261025T200000Z", "generation_id": "gen-1792955930-mOP84U8lGx03kMdoKFzb", "proof_sha256": "e9924dc0ca2e545488fb2a0a35ccd637015156907671a8ddcd112d09f3502d14" }, { "model": "tencent/hy3", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261025T200000Z", "generation_id": "gen-1792956002-jYcdR3wCKLPQFQrvgmNP", "proof_sha256": "cfe67d10cf940942aac3505d57f1f0f0c040b6b29e268c2746cd1dde9b0564ee" }, { "model": "tencent/hy3", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "GMICloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792956024-XFdIw3PKZghkFsViH6De", "proof_sha256": "4f9e641a592bd0e300d5024e098fbddb0db6e459260cbbd92aee75c772b64464" }, { "model": "tencent/hy3", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792956053-Pf295BeG4U45ZiBeNGya", "proof_sha256": "8e71e7f13ffb8e6f936f51a28521f1b06327740a96c5ba7cadf116988c936ad1" }, { "model": "tencent/hy3", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Phala", "run_id": "20261025T200000Z", "generation_id": "gen-1792956081-2RIDpOARu3aJWlZzwqY7", "proof_sha256": "bba82a3bf47d6349035413655ff3cca43bedb6a5103776e8b5f01788a2c6664e" }, { "model": "tencent/hy3", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Tencent", "run_id": "20261025T200000Z", "generation_id": "gen-1792956109-l21mec8z6wS8jig90z8w", "proof_sha256": "abfb5bd7bc35ba449d156e59dbd5c8c0a31091739673d04b92a71cb960b71944" }, { "model": "tencent/hy3", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Tencent", "run_id": "20261025T200000Z", "generation_id": "gen-1792956156-lbahmFC7tA3rCQoal8tT", "proof_sha256": "a41db31122e54396e3f6b8cec66fd2215c9bc3b9be8e51b9687b80cf1fb938e0" }, { "model": "tencent/hy3", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Phala", "run_id": "20261025T200000Z", "generation_id": "gen-1792956180-gGI3D8VJ7S40Sucq0mqz", "proof_sha256": "daf55182f43403e4407027be66319e05496adb1847e79575a17b753883e10c79" }, { "model": "tencent/hy3", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792956200-PTavwnHOa8157ClxnIAf", "proof_sha256": "d891cf4036e0932579a7f45d75e350d3d47f197b19a4723f9d6d671c68758299" }, { "model": "tencent/hy3", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "GMICloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792956213-kkIpHUSj1yGj0rmx2GJN", "proof_sha256": "eb8f4219229242d672adefaa81b2406d4fa312c92537c8f568545b4eaab135c6" }, { "model": "tencent/hy3", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Tencent", "run_id": "20261025T200000Z", "generation_id": "gen-1792956253-nUm8iw8qc0Nxklxn9m7l", "proof_sha256": "9986ee97c0ed3442ff279d7486fd8137ede757b3d75667b250d89d5549fed2eb" }, { "model": "tencent/hy3", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Phala", "run_id": "20261025T200000Z", "generation_id": "gen-1792956282-vtKElyBteZnWdESiaY8X", "proof_sha256": "615cdc9237de259285ed514b6fc77ed488bcb9844d331eb3b1638071cf4c58fc" }, { "model": "tencent/hy3", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792956297-MWH5jeEV65lDR07WG9X6", "proof_sha256": "33148ecd22f343b9ebbcdff643d99b7bda4e58a64aadfd2a7d1188e87f68de06" }, { "model": "tencent/hy3", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Tencent", "run_id": "20261025T200000Z", "generation_id": "gen-1792956337-0FMiswVeif81ra2RYr0b", "proof_sha256": "606c1dfaba0d726dc09d6852a7f6937eea60de404c8f41f05922dfddad7a1bda" }, { "model": "tencent/hy3", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792956357-fGm9cAFrdHfJXGRkwm38", "proof_sha256": "a36c6903c7911609bafd30f82405e58a0fa76b96e2e34209903b24454eced885" }, { "model": "tencent/hy3", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Tencent", "run_id": "20261025T200000Z", "generation_id": "gen-1792956374-bplxuMr8H37NY5SSPIWM", "proof_sha256": "6225cafef03b914faf0018f48f606fb5e27f009881bf0b8106f9b47e398ed603" }, { "model": "tencent/hy3", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Tencent", "run_id": "20261025T200000Z", "generation_id": "gen-1792956429-g9qI0M3SIF76QzimDG9x", "proof_sha256": "81596a565cd9eec64dec5e718457842796d033ce571667725c5838500d1e103e" }, { "model": "tencent/hy3", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Tencent", "run_id": "20261025T200000Z", "generation_id": "gen-1792956465-kiR3WhkvXPgQGz5hyCec", "proof_sha256": "39533bf473d7b21ba3b336133428c096e5b7d2c7299ff0a4a6d834c8af57d0f4" }, { "model": "tencent/hy3", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Tencent", "run_id": "20261025T200000Z", "generation_id": "gen-1792956515-NMmhN0wWiF8Gz8HZbfQ4", "proof_sha256": "14e51c5b507d3c405ab319bbe541580906ad90daf787b914c227e13de0ebb079" }, { "model": "tencent/hy3", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Tencent", "run_id": "20261025T200000Z", "generation_id": "gen-1792956531-dkCx69OLg9Q1yLqVOFXX", "proof_sha256": "4c27c284db18ee285267cf5cc9a0efed346dedf34a467b06427a3d17e3fd369a" }, { "model": "tencent/hy3", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Phala", "run_id": "20261025T200000Z", "generation_id": "gen-1792956546-0PPfrNkvy9KVBdCDnBcj", "proof_sha256": "8843472e3e6426a92806a64621f8d73eb33fc7ec76b9928a948bb93d59e4cfd5" }, { "model": "tencent/hy3", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261025T200000Z", "generation_id": "gen-1792956561-NwtQQ17GaDBj9D3ITdHd", "proof_sha256": "a4aefb920e0b7deb290dc9c4d964b8e035677d07d620f67db46a35e67ee54324" }, { "model": "tencent/hy3", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Tencent", "run_id": "20261025T200000Z", "generation_id": "gen-1792956575-niAD1PP5MupcMl1Z9pqi", "proof_sha256": "3561d19be766d0f2f099ee8c61b5e5b23f8989af2ce22589e2bc19d0abe13421" }, { "model": "tencent/hy3", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "GMICloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792956596-bEOkaRpCllZ7ilbWIt7z", "proof_sha256": "c168eb084bf28424af2813bdd910d02c960487f9102ea5047cf584d71e7c1016" }, { "model": "tencent/hy3", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Tencent", "run_id": "20261025T200000Z", "generation_id": "gen-1792956607-mQm2dbVKBcCZIvX0LdnB", "proof_sha256": "a4759879632467d99bf69766d0a0bfe4f113e38476d36f3dce0710d27f243a63" }, { "model": "tencent/hy3", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Tencent", "run_id": "20261025T200000Z", "generation_id": "gen-1792956620-Y1dtxF3BAeK47loFFIHf", "proof_sha256": "9217b517a665a323a381093c6f21aff25c5d5f63dc4e0a25af0c4a3332c545bf" }, { "model": "tencent/hy3", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Tencent", "run_id": "20261025T200000Z", "generation_id": "gen-1792956632-yBAqUnQ2ocNZe5EXHxKN", "proof_sha256": "fd0ccac797ec95f1d7a3e099750eaf99cbcbbeb6c9485d08908433911df699e8" }, { "model": "tencent/hy3", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792956645-If5MrXjC6ts4yIZORdRD", "proof_sha256": "9cc27c06ed053e6677d93ca1c8a4998e475ccb5eeacc0f38617d8759395bf73b" }, { "model": "tencent/hy3", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Tencent", "run_id": "20261025T200000Z", "generation_id": "gen-1792956656-0fvIPjhWSC11dnhC0Peu", "proof_sha256": "134aef989f4d1de3e3f8b68494c633d1ea88cc756975aa6f1b57e2e4ad7407b4" }, { "model": "tencent/hy3", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261025T200000Z", "generation_id": "gen-1792956667-VhLBGzaoxOgjzm2u1hOn", "proof_sha256": "9fdfbc8e930885bff4884dc89517963f64ec4e51774a97a9e6e5095887f1fea6" }, { "model": "tencent/hy3", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792956721-GOxmiSkEZsBuizDcBgws", "proof_sha256": "10948d16fa12ac9684e43994cf2583b19038c161ac4dd8801eac221ae39d69f2" }, { "model": "tencent/hy3", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Tencent", "run_id": "20261025T200000Z", "generation_id": "gen-1792956732-TLVMjdgMPj9m6944WgN7", "proof_sha256": "9979bb0bc0826f03cd4a3778e33ff957c5cb8c818f73228006db20580a133f9a" }, { "model": "tencent/hy3", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Phala", "run_id": "20261025T200000Z", "generation_id": "gen-1792956745-bEXEHftsr42W44D5OylC", "proof_sha256": "7a89ebac38b682bd0d8c969bf4dc1ca667833c89d80e5e9ba671af848ff30dcd" }, { "model": "tencent/hy3", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Phala", "run_id": "20261025T200000Z", "generation_id": "gen-1792956758-nHldInb0bUvxo9LGN6fS", "proof_sha256": "1dbb35d6c24f9c2a92f99115204e18593ea35696221c1780b273e72b8fc9ffc9" }, { "model": "tencent/hy3", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "GMICloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792956800-3S8BUM2douihC6y4Qqmb", "proof_sha256": "f40a2b4e68f44540fed5d7998917bab124db0fac8d22168fffa287e14a292f23" }, { "model": "tencent/hy3", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Phala", "run_id": "20261025T200000Z", "generation_id": "gen-1792956811-srl2HBYnGx8ljdC24AAg", "proof_sha256": "d560878bf64221c315471c4b61836f42e68acacd74906c05d2b3be5dab2c7d54" }, { "model": "tencent/hy3", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Phala", "run_id": "20261025T200000Z", "generation_id": "gen-1792956833-c69UcINg2pXlIHJ8A3tB", "proof_sha256": "6bd5528cd8d2a4691c7c5fb064675e0f9679e64efc6c94cac83a4ce640647928" }, { "model": "tencent/hy3", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Phala", "run_id": "20261025T200000Z", "generation_id": "gen-1792956865-P0RkUbrye16xaKxlGBct", "proof_sha256": "31ddb2af8ab228c606f1669dffe0ccea83bbe7ab99a463b76376eb90045dedde" }, { "model": "tencent/hy3", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Phala", "run_id": "20261025T200000Z", "generation_id": "gen-1792956892-2JbNx3FHiTIlF6yqH1kS", "proof_sha256": "94edfde4ff69d362a4b5906cb7c3bc3525b3125f1f10cab43e0af13f753b477d" }, { "model": "tencent/hy3", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Tencent", "run_id": "20261025T200000Z", "generation_id": "gen-1792956929-Z5aFOEo20ooZcUlcpSrm", "proof_sha256": "80562c81a010d48b5de2552b1fe4bec1360bc5db60a39e2f7db9c26c70d623b4" }, { "model": "tencent/hy3", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792956950-3BIixNJapVSFLgG5L8ru", "proof_sha256": "768d5d11199bb0a32cd74f558498a56d6a748e40f7059a63918da86660cac960" }, { "model": "tencent/hy3", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Tencent", "run_id": "20261025T200000Z", "generation_id": "gen-1792956961-0QZ1iHQ0X601Y3KXh1pK", "proof_sha256": "9d42e4c4dfd040114d4d9a53fba913e594d78d0b83b19eebd64fab28fea46481" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261025T200000Z", "generation_id": "gen-1792954256-PCPCjrRPoCzkXepKvR7H", "proof_sha256": "f27a71551452295a83505bd3f30128bebf7e58f7b47b3472e2ac13f619caebca" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenInference", "run_id": "20261025T200000Z", "generation_id": "gen-1792954279-I9XTLCT9nlHbHJq2TOEz", "proof_sha256": "eba6783d1668ecb9ef3e8e876f5024f172c8bbbadbbff39c30e9a0a8689c83dc" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792954290-pdiuQT28EiU2mtUcOiOv", "proof_sha256": "801c405591e4e6a790c0e054c583de2102f83aa3b28f59dc9851b8bd8d8353b3" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenInference", "run_id": "20261025T200000Z", "generation_id": "gen-1792954300-W5OeuusPElNLyr2zyOzJ", "proof_sha256": "31ea9e493d4cb494cc2e334eacd0302931bbfa4f4922cd7eb03e26f17ffa6e52" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "OpenInference", "run_id": "20261025T200000Z", "generation_id": "gen-1792954353-w1Ldn397Y62x6iEO6Kpd", "proof_sha256": "0ea31e256ca5565cb29df05680eb8279cb61a1dbda5416eb2bfc651adb06a90a" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792954412-cdU2qeuBgUxxa2yRQzcE", "proof_sha256": "c85c1c3b77607b719b1ec6dcc13816575fc2445ecdc210ed992c2b2a2716153d" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792954420-KregWBwSuOuN7kWrq23T", "proof_sha256": "5f17623315871fda675a4a2469312764f5997eef725bd8d78edad29edf31ab42" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792954429-We31b1lKwqdnQcOJOmJj", "proof_sha256": "45b458e4e12ef43385fc9c8a47d33ef1b2e791e68beb5e0874f6d6d8b22c4e73" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792954438-N43eKJVkbiujIZrk2TzN", "proof_sha256": "0e2e567a1d842846ae1304d9d6e932fbde3b98f7419f9dc829e53ffdce40076d" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenInference", "run_id": "20261025T200000Z", "generation_id": "gen-1792954453-ePfNWI0aZ8KBzpw8Ifp5", "proof_sha256": "e6dcf3355ad66fc071c7de878649115e2275caf0dacf8139282edbf1f95fbd2c" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "NextBit", "run_id": "20261025T200000Z", "generation_id": "gen-1792954465-nCq2HPBdr1ICRfjj84Wt", "proof_sha256": "861cadc5ddb57c10968eba3729aef7ebc6fc71a3196ad6079d94de3b1bae9b10" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792954474-LCv83Rs6wa623Ax6zvRT", "proof_sha256": "01bbcbfe6a3e1658518b1db79cb5a49598c05213fa34723a8b5ccdf9bb7f9cab" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261025T200000Z", "generation_id": "gen-1792954488-c6rFEhbmmuQb1VzCjkkF", "proof_sha256": "434916a6be8ad87893a6575b38c0aece2a8a4234c2346b9a2bae9d97e8a0b535" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Friendli", "run_id": "20261025T200000Z", "generation_id": "gen-1792954495-F5xkKBhC1gF4Fcg2BwmP", "proof_sha256": "f43c2162ba077a29cc85af4ec5932ff995400ce400e85a1c3233550bda70eb74" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792954504-8GHwn61hZMcz7k86RgAt", "proof_sha256": "a439a9d09fde735ec54dbf66f1bbc6fdb7d88c9291f78c7806c262647f650d28" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenInference", "run_id": "20261025T200000Z", "generation_id": "gen-1792954513-55AuCUMl6bHC9AKAeirM", "proof_sha256": "36147d242c972b48387366d5081df8c6a9952192d2af8c39bb87c999d89ddd04" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenInference", "run_id": "20261025T200000Z", "generation_id": "gen-1792954524-7UxIxSOLKvzseeLTjJKy", "proof_sha256": "84866428f40ae3fec43680b3c1bf5562562aa6f032b0f32e12b3425007d387d8" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenInference", "run_id": "20261025T200000Z", "generation_id": "gen-1792954533-N6C5TGnAv7JufIA2Hu4d", "proof_sha256": "6690d89a03bd239a0341b42f1ac50c1ad4f4de17693c157ab4e4e5de4543c8a2" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792954546-bOvoApOrucXCTRxDnFpZ", "proof_sha256": "14474a0088f124296e0157666b3b3edcea6c82bc9a764418b2751591e8da7c7f" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792954553-y6jrdwyjG3lkXPhkPhh0", "proof_sha256": "72f7d3e8031be0fa8a071f7e32903624c0ae1ab71dfe0d943b8243e6d0396a22" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenInference", "run_id": "20261025T200000Z", "generation_id": "gen-1792954561-fGC1b2D499knNzsu5z3N", "proof_sha256": "f8299698b0d7ac4eb30c2ad4fb149e6dab30f2ccd09689337910936a924c8b85" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792954575-tgvJHsVeZrNvnhzOYozo", "proof_sha256": "4d5171f6415be8d1f84197fddedab9ffa0e03856afe39e46da6f4b81a04e9ad6" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T200000Z", "generation_id": "gen-1792954583-FVYxdKRozSHvmCNWWdF9", "proof_sha256": "bc2fc25f671ed0779265793c4eb74aa71d4b5a62d43fcdafff121901a1bf4a9e" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenInference", "run_id": "20261025T200000Z", "generation_id": "gen-1792954592-TP4X9OVZUioNOJV252wt", "proof_sha256": "ad60e9f77b6c285c59578a92a8f37ad293f631302772afb000d3d7dc4dd353f7" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792954602-6e5pEqogsjmiwYjyiJje", "proof_sha256": "31fcf71878ecd09f70ab9cb5b5ec20a7828ca87c764b1d1a542d5c870a702a1a" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792954620-XodBVWVEW4q5JA7mCMVo", "proof_sha256": "a5000be93ffbc7ddbdd0b3955298faea848fc253fab0712c286d9454261603b4" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenInference", "run_id": "20261025T200000Z", "generation_id": "gen-1792954628-JL82VadEEatFUMaEHME5", "proof_sha256": "b3e7ef3a9b90b0402b45f167d565d11e28c5a2ea6e7d3bdcd87c6e81b3e01e32" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792954639-ftyqf1AcFvJ7UAkvQ5Th", "proof_sha256": "57a2628ba395a24166716d3e1ed9d1596b48ac20b191ec18552fe2a076a01f4d" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenInference", "run_id": "20261025T200000Z", "generation_id": "gen-1792954647-bjWe0JWEhkBjllD5yQyE", "proof_sha256": "7088ad4c204b915dd057b84865771c7ba7bbd53a117b61710664e14f48c0d447" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T200000Z", "generation_id": "gen-1792954662-HyoD8Ka4vl4C4Vta6Bjz", "proof_sha256": "09a2ef519ee847029304c5632d0131efa4b0032005707194bb6d79b64e2f49c8" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenInference", "run_id": "20261025T200000Z", "generation_id": "gen-1792954670-KIXg1dX1IFAAqoR7icO0", "proof_sha256": "ff93063ee22f59a14b0444c18c845b6b3305781a25671757aa4333e9e94f79f3" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenInference", "run_id": "20261025T200000Z", "generation_id": "gen-1792954685-41w4KtSNvKWLvQndjgnE", "proof_sha256": "dc1a283fe808d1c0564ac0400ce5d4e5ea00833026829bf5485a2761745c357f" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792954703-RoZksNLa7MOtnuNwLajW", "proof_sha256": "4b76f92ec80980be920b6c9868dda82cbce0693a474d5300f31201df6c39c76f" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T200000Z", "generation_id": "gen-1792954713-YT9kp769qfN6U44BV1Va", "proof_sha256": "98185e701542f95d0dc92a08b6e0402091007c8388a097c7435a9cfc3e20b20d" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792954723-tkNaVv9KO4KFz5foDiBR", "proof_sha256": "f2d8b9ebbafab5fd2bb8392416abf260e0b3c21bf475755e08b436e55bfdc33b" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792954731-5NCnEQEYA8ulWSwhwIft", "proof_sha256": "ae24e2c5308fa16115b00e641215908a5f9c82c78f76a0059eccbddcf3db2250" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenInference", "run_id": "20261025T200000Z", "generation_id": "gen-1792954739-7Cs6DbVsl9h5kA0m2ekp", "proof_sha256": "0f81e02256ccc62d1cede8f699e165884b8fe5b229fad3eb2acecc13b4857abb" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenInference", "run_id": "20261025T200000Z", "generation_id": "gen-1792954757-fKNmTYD9P3ihlv2QCrJv", "proof_sha256": "61acf03543986a41dc6c55d03e3e9c2dc2d2768029767a8a468565f48041613c" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792954767-j6hPGekbnc0iXUN6Qr80", "proof_sha256": "8a3f596e7dd67ee0d6b170324a3aa2e6eca3991dc2e72cb08557480ebb2096b9" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792954775-pbIiU2dxd8zbotvdZYe7", "proof_sha256": "5c26096e749434b9fb4bd2100f1ad1406957cd46104c5c3a7033606e10c76203" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792954782-Y7ZVWsPJFl8o7VaHEYok", "proof_sha256": "dcfcd3b1a2c97c6e002fb4f196bb7f5fdf01f27a0ce74e6ad2a6ad8fbd616366" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792954790-WnZqGllzJTIR8FMjlyHq", "proof_sha256": "1cabaac232cf3085633a1403cb702b0e01353e3a958107c472c240f5451a546f" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T200000Z", "generation_id": "gen-1792954798-dz9fAfjPsCEXFYDVwta8", "proof_sha256": "5fcc54261304f168beece095e04d6830433e3e551944a883fb89d817009862f0" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "InferenceNet", "run_id": "20261025T200000Z", "generation_id": "gen-1792954806-86oNaxdJ2H7UtGWAn0fV", "proof_sha256": "0e438648999d900d6afc7c1dc15fc0a695cd30e17bf9b4bcf4803fd72002ae79" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T200000Z", "generation_id": "gen-1792954853-4lFXSuc8hmoP76xVF5oE", "proof_sha256": "1e04717fe3fb946cd20af6ec1157bb2de8ee9b467d5a412611e93c7eba1f921b" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792954862-eKaNYvj49Bq6yx4UucBW", "proof_sha256": "b11397f74f112a54ba37727e0b05e7b6ad316047b3623f9adb4296d61cdfc50e" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Friendli", "run_id": "20261025T200000Z", "generation_id": "gen-1792954870-028vOC0SJOkMFdLkjvxc", "proof_sha256": "616820d418748093a908173aefdb6962b4ee859bc1c4c2163a34e45e42b2a8ba" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261025T200000Z", "generation_id": "gen-1792954877-S49vBvBHlCtsNDL8XC9g", "proof_sha256": "744de50979a1486c1e4c45bd054bc4fa0ad8c44c3f765a1254ee4e7974e78c16" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261025T200000Z", "generation_id": "gen-1792954901-KgQcp9DQnOe9TFg8kJZE", "proof_sha256": "b244ba8b2f348844dc7362b0809805cb4843aea2c65dc7955d5d7381a7a6b369" }, { "model": "z-ai/glm-5.3-flash", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792954909-9u1BSXUvsnTa13zRdJrl", "proof_sha256": "50739630ddbc1e3f4bbcdb36e85ef49c1169a1ce807f80c45045778a15239595" }, { "model": "z-ai/glm-5.3-flash", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenInference", "run_id": "20261025T200000Z", "generation_id": "gen-1792954919-7Q3hrqoDSVsbV49Td7vF", "proof_sha256": "a8aa2d1c28610564b357ad439d5a86257040aa05c49ab885fd7a882402a77ce6" }, { "model": "z-ai/glm-5.3-flash", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792954929-7x95o26oHJEsYpS5Mer5", "proof_sha256": "909f20626a46bc434a3d328037058ea18e5718a901f4af5b3c7a39644c589905" }, { "model": "z-ai/glm-5.3-flash", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792954938-HYEs9dGKx0wE6hbiQKHa", "proof_sha256": "db3cc6c5426581c0861c02d4b373fa2a9881379edb8b4de08c586871da980ffd" }, { "model": "z-ai/glm-5.3-flash", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792954946-mlnhjZurPFMCurE9PLpn", "proof_sha256": "866b7d53680b8a71a4e531298ac7dbb7dec4931a1e23bdf1b3716ad018b4de7d" }, { "model": "z-ai/glm-5.3-flash", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792954957-5NE2G4lY2Z1NzULnQyzj", "proof_sha256": "8bbf934ef9d1d53ec8cdfbc4f44d25fdb65b33d1a3124de78a352428db823871" }, { "model": "z-ai/glm-5.3-flash", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792954965-zLrIkOjEBl0ATKuiKyLJ", "proof_sha256": "25d7da9761652481b171e1497ec0ab8c7552bc70920f7e8fd4bb21432ca59955" }, { "model": "z-ai/glm-5.3-flash", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenInference", "run_id": "20261025T200000Z", "generation_id": "gen-1792954973-Voe1okqcBfPc1KvxENKG", "proof_sha256": "f2379709bb19a1a2c6f3374187310cebc5680f96e7f4af5c82ffec0752a714f3" }, { "model": "z-ai/glm-5.3-flash", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenInference", "run_id": "20261025T200000Z", "generation_id": "gen-1792954983-6KvfbK7f897Eyr6DNLF6", "proof_sha256": "9f1f7acfe2667726da676fa8a7d85fd73b2cee89b849b0e9d86343379b77361f" }, { "model": "z-ai/glm-5.3-flash", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenInference", "run_id": "20261025T200000Z", "generation_id": "gen-1792954991-065g7o6TNRJZZrgNJ8HX", "proof_sha256": "3913b7bc100280a25f646135bec05f2867d09eeba2f8cdf9f6f1f531525c0a0f" }, { "model": "z-ai/glm-5.3-flash", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792955000-gOE5jpgttTrffEpgyBc4", "proof_sha256": "35190b4f80d907fd8e973affb154ba5b6f45cbda70f4c9fd6af48f0f0ffb2d12" }, { "model": "z-ai/glm-5.3-flash", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenInference", "run_id": "20261025T200000Z", "generation_id": "gen-1792955011-UbjNJk8Dx0tT6toj9n2K", "proof_sha256": "b030123d92aeed2bd2df47b6e531cc2946a7681bb9456d92d093ffc39ee6574e" }, { "model": "z-ai/glm-5.3-flash", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenInference", "run_id": "20261025T200000Z", "generation_id": "gen-1792955021-CV6vQ1RFwuq5HIJOqMhR", "proof_sha256": "427360196def97087fb648b4a616fb3e4c0887310982bf1df1141e32c114ea6d" }, { "model": "z-ai/glm-5.3-flash", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenInference", "run_id": "20261025T200000Z", "generation_id": "gen-1792955030-xw487DeLbQlTDzeud7YK", "proof_sha256": "1a5a05fa3c0502c15ba8f42886580219f063e189ee6fe121bc5d43cfc8bfd7f6" }, { "model": "z-ai/glm-5.3-flash", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261025T200000Z", "generation_id": "gen-1792955040-wl7zsZt3VmJ337V93xnK", "proof_sha256": "47935d1364fcf90d2463abb5cbea8df1f92b6f96c3e3d3c46855685b8c75af9d" }, { "model": "z-ai/glm-5.3-flash", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenInference", "run_id": "20261025T200000Z", "generation_id": "gen-1792955047-I5rflp5ejxAtX9lQ6UC2", "proof_sha256": "ab207778e06d23aa0f2a7206a3527dbb5dae0d70cb91cc3577f10c79456fefd7" }, { "model": "z-ai/glm-5.3-flash", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenInference", "run_id": "20261025T200000Z", "generation_id": "gen-1792955056-5ECyGvR6b3GuOF4AU5zA", "proof_sha256": "88e68206ba47611fc6805904a4b71c523e001ae3d4b590bbb688f9849d361648" }, { "model": "z-ai/glm-5.3-flash", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261025T200000Z", "generation_id": "gen-1792955065-1wgXeV6GM4DTJV2fTNDm", "proof_sha256": "9025f53e63290df20f9f41eb3bc24c648c3c4c7a7c1db1aa3cc3aa7ba8746b9a" }, { "model": "z-ai/glm-5.3-flash", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T200000Z", "generation_id": "gen-1792955073-J8DNj8AqUrNOFTQRsqmn", "proof_sha256": "ee644024d284633baf751e609924f8f7f91bb4b8e452662eda4f60b170bc65c1" }, { "model": "z-ai/glm-5.3-flash", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenInference", "run_id": "20261025T200000Z", "generation_id": "gen-1792955081-Ftaplg2KbilYgDVkoUPx", "proof_sha256": "c0c6cf5cf0475a739b3f9d0c9f2bac24a25e398cb7e46ac4dbe9457dbc6034cd" }, { "model": "z-ai/glm-5.3-flash", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792955099-DrhqO4VhQhedU6uyDG1H", "proof_sha256": "c56d15b73baf0cc8df6039f2c5d576ad098ad6c4d485dce9a94b88f2b267a0ab" }, { "model": "z-ai/glm-5.3-flash", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261025T200000Z", "generation_id": "gen-1792955107-C2ygxawboqoWSKRTqFhf", "proof_sha256": "9e645390cb1205372907f64415123ba40ccddde2cc4f04e59bf6dc0f3543ab38" }, { "model": "z-ai/glm-5.3-flash", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenInference", "run_id": "20261025T200000Z", "generation_id": "gen-1792955114-1ar0llXmeGaifYcP3LcI", "proof_sha256": "d5a418f6d60c5c6d44ae9f1462f3442e2e92237c453e44cc406743c2a7284904" }, { "model": "z-ai/glm-5.3-flash", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenInference", "run_id": "20261025T200000Z", "generation_id": "gen-1792955123-U4kPnBvKaBn8MS0Q5VkH", "proof_sha256": "347f8398bef49e4efb6a1d3f20240da7bfad651400ceb52b6d1eea7f3d45b9ee" }, { "model": "z-ai/glm-5.3-flash", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261025T200000Z", "generation_id": "gen-1792955132-FLDN23udrt1EJdffUvNa", "proof_sha256": "b20746723b2d6220c20ee1ff596307fd6864c85585d4e602a0ad498599ba3f27" }, { "model": "z-ai/glm-5.3-flash", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenInference", "run_id": "20261025T200000Z", "generation_id": "gen-1792955140-pLeYsYa0FDZL1VH2bzR7", "proof_sha256": "abd604724c7249192438ea3dfb4fc809044899cb480f8b80504ccdbce7a8d5de" }, { "model": "z-ai/glm-5.3-flash", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenInference", "run_id": "20261025T200000Z", "generation_id": "gen-1792955150-fi7ai9Y05OYkaJYThXdA", "proof_sha256": "71bb8b22d66983331229f0255a0a96e2d3bdfe084a0c04039599db181a7bb404" }, { "model": "z-ai/glm-5.3-flash", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261025T200000Z", "generation_id": "gen-1792955159-sBA0VKD1M65Ap1R5oZqd", "proof_sha256": "8161c32367aee5c62272219459c5853e0854f4ef2a122114956a76d8fa8496ed" }, { "model": "z-ai/glm-5.3-flash", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenInference", "run_id": "20261025T200000Z", "generation_id": "gen-1792955168-6kVynag4zlQikUgdPvTr", "proof_sha256": "250b975386f07be7a838eeef7ce360d078c9b8f9ac1bec99cc412d6d40e0526a" }, { "model": "z-ai/glm-5.3-flash", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Cohere", "run_id": "20261025T200000Z", "generation_id": "gen-1792955179-NvM8IuR66EhpNM6q5uul", "proof_sha256": "137da0b418f70d426719b6cdd37f21ec34332a0fafb07109ae611beff83468ac" }, { "model": "z-ai/glm-5.3-flash", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792955186-wGugrJsb27hJmNGSB7U7", "proof_sha256": "2d3892cd307dc56d01f2320a8d676b4169ab467a31491f2277ede3a87e1318cd" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954256-JM0pCP88jy8s8KV3pujc", "proof_sha256": "e9dfce4bd13d9e4493ad7dd7ff48f4836f2373f70b134e584fa3222bb2803132" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954304-ZWHSjQ6KR5sLyf7RI9ia", "proof_sha256": "778aa90ee9a45ac3fe705c485afd895f71632e93a9e1618358c282a23650b29c" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954353-rynUZLYvo21vgh0HefZZ", "proof_sha256": "2291e896f5b947860afc93f453828a4477f76c5d2620a5c32bf66efe12ef6be8" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954373-kK5QiDYSBCMIGFCElRe0", "proof_sha256": "280dd8021a75acfdda86223b573ce0e31ba49fa5e5a5ce69d3a93d331de5c480" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954417-ra3S7z0syeKT4YpqJt0F", "proof_sha256": "9ccd7a18638dab4034aa7a01b16dc190bccbc438ac9f00c56076183fe2b6743a" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954450-vrHHp7A1WvTBfamgKsy1", "proof_sha256": "186ec341bcb4fae43b0bb224cdac28955ef1a95c453d8f418976cbec5f901ae9" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954491-rXzUsw1i5YcVnq3E9aLr", "proof_sha256": "a4a15d3583a74447a94da2ccf32002fcfda62d3a61952a9d11dbcb37c927fb62" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954506-lYMm7Ghs58qSOv1EZHCj", "proof_sha256": "b41eb0d9a769fa9cc67dd46eacf2de19e81ae4bd87bb25d9a3f718d39e242e17" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954528-HmC757WNCQ2mfdeseg6C", "proof_sha256": "f7b532779129a019453012ffb09a3597fab961752a3f5e9286d41dfb134cacec" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954573-z5zQwBUWW4ZbOXKUfrhr", "proof_sha256": "ae4ab82e872aea1fb5afa18cd4b0c61860c859bcb9a9c510e0bfa67095f0d571" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954607-OJhv0A2PQ2UZDAxzn3Fn", "proof_sha256": "b864db5d75f3298d7ea132b45fd8f4b0dcda66d12a4e4d181351272451ba3f21" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954633-vSoFkI2SMtO3wp5maXnP", "proof_sha256": "01a647d508d4871ed1a5ba8c1a2327112911b7ee6a8ad39d99e7baf00e6807be" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954678-HAHgrlpHzLjdvXOBl3ME", "proof_sha256": "53204a462d12e0fbca494873ed46656ae459f3c98cfbf5814d52deaf54ef0712" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954710-sfb90srVOCaCHsjBZ9qc", "proof_sha256": "6c22c70fcc871050c5eaaf93d5a530f45db95a74abe29343e632792b169678b6" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954742-Q1gBrCRdTt4TwNkyxNbR", "proof_sha256": "39747ab1aad672b1f2f477a0fc7cfa818ca86dff9b53e638d8bec7a0eb6302ef" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954791-tueR9IOsn3ZuXkUL6TrM", "proof_sha256": "b0b9a37d1f2c0dd8ffa8840fb54d4f283d62447df4ddeb9eeaa9c4411d8b9f97" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954815-zCRpMgqyZHh6o9KL9MLg", "proof_sha256": "b963cd7c14774a9477ca670b99ee633682c2b94743a642174d7049df25e43bf3" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954854-fKjvwQPcuZLlr3TnvjbR", "proof_sha256": "c0a7e3ed4dc57f614570883fffcf76eb1216f21bdefbbf40380a28df1234e217" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954893-LxiRIq2ustAAv1jFRZsw", "proof_sha256": "04aad22ca718b223f2b24adeb9f4d0dae47cc0337729b5beb0d0e1c13f707301" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954932-S7lVHYn5BmxlI76Y8Ppa", "proof_sha256": "9ce3fd037cb58c8829e8a470bbf5fa42d6d4c213d04f61bc16c6c124e49910cb" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954947-muaDCJSZ6QGmrqBNOqng", "proof_sha256": "abce9420bb2a64cd3585c2fcf2297116052621087b1fa0371c840073f04eb24f" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954972-0s9YjaTuig1SVQpym2c8", "proof_sha256": "77520793cb29c5a8a67ad3a1d8376ce8c112bd935808cdb9bde388945865ba38" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792954999-LiTO7LMBmuLc77nzZ9hY", "proof_sha256": "f0548ca56d993293d3631515164bf0b89a074cd903cae035348433bfa8dc018f" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792955033-wAgis5vWh1e3RndRr8JN", "proof_sha256": "d29f84a5e7d3e50572ad0615fd5062907a12a0b07d1dfb4c49ef2a09a40376ff" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792955098-CldBTCERCH9Aj8qKMAn4", "proof_sha256": "9aa24162abde3e50c6f9490517116b83009bdc312e359c681098a249e5c6e426" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792955144-6SpNVrVpUDdLxO3vFfkV", "proof_sha256": "c4b7e2c7a29a7fd833a367f864319b7811380c372af4d754a17c897b797b3977" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792955205-VIkopXAc42w7dmR94L7s", "proof_sha256": "4f9d89b798f948e923463a22577c3fa41dd4cf066e83de4c85f0355e9a66b710" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792955253-1wZ7lhpB9iMOvG9ZBmOG", "proof_sha256": "b426013e9fa4a42f2c98682e75f90e55a0caadafe08fbda992ccbd6bf868a55b" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792955283-nESubJbWfAuRcn15LWmC", "proof_sha256": "9ccf4f5533b76506ede5e9d5296669b2fa4af188e79dcad61838780bbd525713" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792955338-ArshX4LDXMcjSPy2AUc0", "proof_sha256": "d8da356b2d34bb06dda3ab80e7be5695b9277a50ec6091f7f4159ec064d4b214" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792955419-LsPSoCsPAvUBrOt50Vw9", "proof_sha256": "4adddcb2439e185e813f897dfa9b82a0812d0c180ea761123cf3bdf826e4bc67" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792955491-2mlVBCXNi9QI8G3RgBcf", "proof_sha256": "302bfec2bcde78fbba4ec3349a7533799e13001b367a7117ffb6458c81ecfa2d" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792955539-UbwF2g4TZPlsOGSusRQj", "proof_sha256": "411d4ecb8d979f1ee535e247abf04ab3afa73f366baff9594f6410cce5f4ef67" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792955621-0Wy864C2bscZ8bTVyW7x", "proof_sha256": "fd90832b57137e17308a237ec1b722b0f8cda56db6bf57a84bfe3412b79e8a41" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792955678-pwsRtXrqpCG31PZCsRHX", "proof_sha256": "c47a0d3af176634ee51699aa19eb9784521387d9b25bd2f09768824713aadf04" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792955797-ayFCcT5L5pxgimyVKXKM", "proof_sha256": "43f07ffb71e5fc6a1ce59df8332e65eefa3441e5b486622664931b1015a967ab" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792955880-GfNrjC3ItyFPIXBiS1Z7", "proof_sha256": "bc379fc477f922734c29008ac7a577e7008d8e732ff5b51dfd8f178b6362fe8d" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792955951-miYbochzR5x9MfrYk1Fn", "proof_sha256": "e652e9071bb3b6bdfeba4edde7831f25f44db811278347240810b3d178dbf17c" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956011-uJuMS7PtFbjAZ2x98tgk", "proof_sha256": "47115bb18fe185cae78cf338a8a233a81fa5d315491557bb5a53480a39f44459" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956066-AmNshCXIUQbHC6nzyutu", "proof_sha256": "19754817d61db4e7a4639a930798b75dc095d7872d6bf8842a7ecfa4c4ab9c38" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956091-nLa7L2bNmyMTuWDiErBa", "proof_sha256": "766d8c07e87df076c5b14cd58de8936e638edd59d55453a069b82c8761b963eb" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956147-ZolB02RWZwP93IXcp1Mk", "proof_sha256": "34aba01ee3ddbf17e2c7ecc9ed15a2f0c8fd9df74ee99fa1f3bcdb3391875e6b" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956199-w6saGSZenOCx2dPENC09", "proof_sha256": "0a3ade93aa9457c9a5324dcc22000c52df9188ffa2529aa40a48a62d584425d2" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956234-WQ9lbx0ioi1zHJ1wDsFx", "proof_sha256": "7f9f356d384e50b08c82f462506fe5f592fc431b15d5c5c30c3562c5c976b930" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956288-Z5gkopJNhWYAcB469Att", "proof_sha256": "6ee668345ea1373d220f569ce76bc46f3b3e0629a999696f64518e9997c5a9d0" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956318-PhhUCapjseg0IPCCuIQr", "proof_sha256": "3bb89b86814579d34e764f0faf5ca182b6ec884b1910af146de8427eeaa3d2f0" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956363-XAIJ323cgV2JNiRq3iMU", "proof_sha256": "1f6c1afa7bf3a198594ef9353a2f809f720743fe042d1b64268ea4c9ab9f68b1" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956423-m2PpyAr6P0QwmWqWunRv", "proof_sha256": "e1422668fb0d8da28383c7cd5f746313221f4d08dfb413cd797f2097d5eab568" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956484-QIRpMHqok8fOpCOB20BS", "proof_sha256": "35b8f0725bcac508127d6eedf124282900ae2eb4b35939aed4da20733711e9c5" }, { "model": "upstage/solar-pro4", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956553-5j9LNe60og1FQOJF5S80", "proof_sha256": "269658a99f925d9af31cc78a7a638fdaae70c0b277651287a022b02ea1191035" }, { "model": "upstage/solar-pro4", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956580-8rq0LUhkkNzPmCQQcK6K", "proof_sha256": "6c01386a4f8877fb4b6115d2ca0d621104ccd969b061d1332af4fa5cec4026a8" }, { "model": "upstage/solar-pro4", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956644-a9TYxZTkgAthVnnRVFNq", "proof_sha256": "3a1590557d5153cdaf2279f62213eaf71c604f1ed2d9b0ca471caa0e682590de" }, { "model": "upstage/solar-pro4", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956707-CYiXcJewpBdbBhwr2zk7", "proof_sha256": "684b8c2acdfc04c8f23009f0526244665e2597d35dbe5c72fb8a2c4c55d6c6ad" }, { "model": "upstage/solar-pro4", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956758-35sYgBPbLz7fJkKdhamW", "proof_sha256": "933bb1355afa449a3a0affc36c2e8ca2ecf1ca895aff997ab5b2eddf7e201b4c" }, { "model": "upstage/solar-pro4", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956817-mKoHiLvEqRSgxZ5gEY7i", "proof_sha256": "41ec81974ec097e11c6b68527135b1dbf7935c84a8205256394bb39432f6c523" }, { "model": "upstage/solar-pro4", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956862-oLaWH8opb6Hp3iFXzTbo", "proof_sha256": "02f649df411b93d80297dcd1e9908182df1aafe96bdc57fa65b2b28e3c45bc5d" }, { "model": "upstage/solar-pro4", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956880-H2D8tE2MAA2FXmZh0AWg", "proof_sha256": "b4e5cf38aa6bbdbaf9eef811af1cbcaea8f83e56b3d53ec5f80a46f6adf118d6" }, { "model": "upstage/solar-pro4", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956925-94v332DOUcxFkJPvZi51", "proof_sha256": "e6a9edc2766fd333713d82adad5a9f9b45591a7b27bf9c24d8cad1e72e846604" }, { "model": "upstage/solar-pro4", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "3. conclusion: to see if the right hemisphere is processing language, one must observe metabolic activity in that region during language tasks. among the choices provided, the", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792956958-DhwthGhhLh4xjYdjia8p", "proof_sha256": "ba5120b08b224958549597492149a9a304388d787c354a1bddd855b74f8a6682" }, { "model": "upstage/solar-pro4", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792957022-a7YyttsII0vSndW0DudI", "proof_sha256": "bdb85d5b5cffc72bfc7609a964a517d88b56f17fe9d990f8c93bbd1bf8d76d1c" }, { "model": "upstage/solar-pro4", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792957049-F88WSfY2XoP8IvLiA1MV", "proof_sha256": "d854be2d87c0c5d25a2fef1010f0ddc03ba8b1cf0d5db0edc3584f151e34e85e" }, { "model": "upstage/solar-pro4", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792957093-EM62eopKllVVC6Wr3AQ9", "proof_sha256": "2ed15447e850803d3592e25ce1ce0bafbadd88a1b102473469326d44bbc79666" }, { "model": "upstage/solar-pro4", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792957124-w6Y96tc6kAyCiNcGsJd2", "proof_sha256": "617c2f878db56a10220d7d4bb5cb6b398f30df4aa11b3be753cd70b754a29ae1" }, { "model": "upstage/solar-pro4", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792957163-meHLNDP9FBHcKyXF35ux", "proof_sha256": "aba9dc9d26227044743c1ef63dc33df085acc5890755cbebd07c3f219e1fc845" }, { "model": "upstage/solar-pro4", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792957190-I6WvcfN6geZDTEcp8NKh", "proof_sha256": "41a875d1bd523b83a33b84be240174693381d0858d926dd680e91918b36d8a07" }, { "model": "upstage/solar-pro4", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792957222-wPuxSBNPjhAmnubWF82W", "proof_sha256": "76d94ac1d2b9ede60f50d3e3c3105727308f234956e045d77e86c6068842d197" }, { "model": "upstage/solar-pro4", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792957252-M5jWFbgAjdYWtTC1Y7L2", "proof_sha256": "f0415d48064deacfa99e718093189e33a9c170d9724af63193b9206b1ad952a5" }, { "model": "upstage/solar-pro4", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792957330-6cljXq2cW7QWAtDAiOym", "proof_sha256": "48938324cb74e74f81adca3f2938c0e5470968bdbb11856f6ee7ae6af10f6096" }, { "model": "upstage/solar-pro4", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792957352-4BfXGf5rQ8r2w8h91Q6Q", "proof_sha256": "546d381c5771708fec7525c1ea6a0beae936f4e93fd141e70fff1b31a75815ba" }, { "model": "upstage/solar-pro4", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792957370-6lvOQmrbzk0mam0Cs9tB", "proof_sha256": "2481186e9c47ba5f6d85f3b068e2da2bb914b168412981cbc9f775ebb9a84d8b" }, { "model": "upstage/solar-pro4", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792957415-Xa9kvAlGZAPlE3CqDjam", "proof_sha256": "c47c97e3251307050e84ac49939bb470e4a801d3bac056f59dcb23792d95469e" }, { "model": "upstage/solar-pro4", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792957453-Z1X0iPjBDGxgljGVpEVz", "proof_sha256": "9bfeb61ddaf332795e35f31341c8210d399fb673a0bb894fbdb7791d44749a1f" }, { "model": "upstage/solar-pro4", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792957510-LNYMPYYLcNLvEKVhlig7", "proof_sha256": "507b21585bc2f488e0c95d23108763fc812d48dbdd11201a8e3aa64e9b7049d5" }, { "model": "upstage/solar-pro4", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792957521-ZlpkUiCftgVAJN3UvUiJ", "proof_sha256": "58a45c5fd1c84382975be4eca3b9d72160b4b9fc02cc9745adfe0332f6570697" }, { "model": "upstage/solar-pro4", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792957566-pVHsKRzTMz9VpVGcqvlx", "proof_sha256": "c67d47430c40d174172b234fd35fd58789236c9e71849fcc39392e49ae357a27" }, { "model": "upstage/solar-pro4", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "the wave function for a $2pz$", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792957634-KBh8RH8TVu1NCOVsAqI8", "proof_sha256": "daecbc4e25175ea1eec7fef982262a107fe9be162100781c5d1a008745ad984b" }, { "model": "upstage/solar-pro4", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792957701-JjLYiPMLium6mjAXjgYD", "proof_sha256": "471eb9785b682f3392175cb26b2f98b59817ccb7b91aa69bbeb29ec32220d6f7" }, { "model": "upstage/solar-pro4", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792957752-roOV3BuRyii3SikzHL0v", "proof_sha256": "dd727fefcf8f8c90c111a8632d6e932c07dbca52bce5dc4fe83c999e5435efbb" }, { "model": "upstage/solar-pro4", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792957766-5RPj5L66iHqZN28K3eln", "proof_sha256": "ae03292703caaace3ade02e23b8ea7066e66d1d422f3e80556fabb327114cf15" }, { "model": "upstage/solar-pro4", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Upstage", "run_id": "20261025T200000Z", "generation_id": "gen-1792957787-7Xq7d4edEZag5VZ0o8Tn", "proof_sha256": "3b87d8b78fd89ef6c43f73ab9c77ec626ad6e0a81e5046c6ae64b218f33de91c" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954256-uLCOdKNFaaXzCpNKflJW", "proof_sha256": "9efbb1bdd184d216452ea246925245139b4502be8e6b2bb9713ef0a4cc271993" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954284-reP0hY4SGj1iRDoVAaY4", "proof_sha256": "75425512e40e8699fac6b003ee58fc80dcf195862315ca27f24e1e1769e017d8" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954323-LS30OXmhwPi4t9sagcd9", "proof_sha256": "0ea8025f15cba0728d3370cc0baaaa1cd3ddd5fa8f374cb915fe7ec0b8824bb0" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954350-pwehPXMJcKNn3uCprsVI", "proof_sha256": "344cbe5b14ed6abcb9bfac580dd5e77a63fb2cb5dc6f33b8bef74bea5b28cb6b" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954377-9KxGPpWhSX1bEJBqRc3c", "proof_sha256": "1a6c70f61df3b70edbafe26542185d8edc0a2c228f50cba6e0459bf1f81e79d1" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954403-IAPCLjERNth1mKUT2hj0", "proof_sha256": "b9809b938d6338f659b8e50ea0833f5c14d5539ddb88a54de2317d1d180226ea" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d} \\]", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954443-N2tbFJzKPoIEzloSil9R", "proof_sha256": "3145596f2b0bcaee3fff4a120aa830c2086b90a2de35e97273585a7f30f23153" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954469-1wA6gW2ZD5X9jk7FeHbF", "proof_sha256": "177270a2aa9952c62fb94bbe8422d161c34c9fcb3e9a9e1a296fb1207f6a40ac" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954501-KSMqD0aofkmklEp3T5n1", "proof_sha256": "bf5986160101c776802b276839beadac5e72a51ebda07c36d3e0ec71e65e3105" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954527-tj2SdAiqnopRUJHHqyoI", "proof_sha256": "8d64a565978d2ce6eb6cd1cdd938dd3d254e8b2603f6db8abcd7edbc0da6a50b" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e}$", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954554-kklsZkNQApqUI5mDqv2X", "proof_sha256": "dbb052a57d223179612a388812d836226300adc2571b54126aa4d48319de2f9f" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954578-oxSpVX8xC0RdMST4jbWA", "proof_sha256": "29c02b5a75d575d50e531569833d19567be645b70e023377b1e4bc925c4a9deb" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g} \\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954608-Km3Pv3e2VS5lJMCgeQmM", "proof_sha256": "e679099c26cd06cfd7ebac707f4e799130f5972a317a924ca9f1225ccbffb597" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954627-J6ejelX9DfzRgfq5Sfiv", "proof_sha256": "9b46c9d403187f81964dd75ed49e8675bfbf54936094f6ee1767e82900af90bf" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954653-I2VUVdl7WMqqXQTCra3v", "proof_sha256": "565862a5e5a775903f4a251130ccf357d814d2493e2cfb1c5ee9f8822c8fa59a" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954679-hQvusImMAwYD6dJqHGEB", "proof_sha256": "84658d6bea6b1ac8249912bb19f817d3b10f03e9d3e8d62d0ef3e15fd6e76209" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954705-Tlu1EIfYw7LIQ8vQ92TU", "proof_sha256": "7472a3aa4650568891b725cc42055ccd40ab619c4546393b4c45cd1e6294a4e8" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954737-oEic5mIiCnNWu4wxrAgi", "proof_sha256": "fbc208853251ce52b488044a67e966a1bf69a968887a3c87461386ecddd720b6" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954763-l23nckP1Wvve9UcNwOUW", "proof_sha256": "dc41b7178f747c59f96939ac3bd57f1a8bbd95d31564e71057748b7dd04a1497" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i}", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954791-jCWZjxJLpgKA56CZfgpj", "proof_sha256": "90c0d44f3967135726b9df9469b86d3b0644111347c1465b0c2bf3e46d41a505" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954809-kOsUGdyUmDP4PBmpgdnc", "proof_sha256": "5e9f9d6909210554b4926839f1afa118ee85c05201638120f949876f8ba15da6" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954836-bfeAifD2klUFn0sGHtV4", "proof_sha256": "435dba6f056bc8a49dbe41c7f9c2374b8d84c5f9bc4632c988c1220f75d5ee09" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954864-V43M0SlzC9wzOuXPnpW8", "proof_sha256": "3b0bb123019299d893c504b32ee7d6cf28b4b2a6fdd48522b546c1f1b7860169" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954893-ke3vJs47u3VtLM3KvDxX", "proof_sha256": "a04ddd297351afc2d834e0e61f81791b49ac0b1790a118513783376e327acbf3" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954920-czvnP2DpGxkqPbGE2G9y", "proof_sha256": "2fd3915a1a127bae21e3a34de670ae67fc5abbf575dad53c9f6e8ac1b341dc8c" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954949-7nDWtLNmZxYmLIKRs69J", "proof_sha256": "a166c4ee0d5f19fcb1776f55962825cbe894cf7f3c59be2cda53941b21785b7e" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792954977-4qDuBZrdJHwEqMyezGjF", "proof_sha256": "be22f8440d203372eb532442d10b00bc88763743d425798de4b40d7367d11597" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955005-xFfEsNChNLkfpTdxg3VB", "proof_sha256": "56cc7a84e8017f36fb2ce800d67784eb21cf3ed904ff24904f01bef7eb376820" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955045-0WLcsP8YabPQKOtFStbG", "proof_sha256": "a628de019821220e8308d252daaa89da1faf6f4c09d9d53ea7b912fe551e159a" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955072-F2vxblIyz9VZG5x9KFM5", "proof_sha256": "c0fe628ddc2b1039853b5afdeab0a7e7ee3af8810ea571f67a3b0843c213b3ff" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955101-A27LyHoJorQZukTesesR", "proof_sha256": "9b2ce6108d1c0c28410399299a4352fd98bae8cb518e00d4fffbda07b65432bd" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955128-CJKsVQpHCrhYcGKOTGv2", "proof_sha256": "13ee97d479efbdb5adcd51e70088d7225d541c2b4f0bca068f64a37423afcbff" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955157-lGqzt1X0lnNPrRxETOuW", "proof_sha256": "809be356fca194ecdda138a51c42a7e9db094bd5f5169723a8e20cd95fa63bfd" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955184-7up7fdNocvGFks9Hxksf", "proof_sha256": "5896ab46077629927f8710b57e623aabd755c056851f9a2ce293c5e4458ec804" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955226-UFNykh0MYnrKfJMrTnmA", "proof_sha256": "88a7c9e3e14618516ed48c192147e0d2b14287fd62d64286ebb9aac1cca51b98" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955255-F2oHoAKUVKxzcOrI0RDw", "proof_sha256": "3513821c5332e3c49232b36c8d6a63286f9faed3e261393b6d8143aedbfb2da4" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955282-tNeqK25SqgaWzqAi9INy", "proof_sha256": "56c9cac461eebf21d0e78f581de0dcfd4d3c04a7ff01604787d68999fb61f45a" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955322-gRK2VOeQBAIE35VswUI5", "proof_sha256": "61586aa2ee29a5eec4f214342266b5a83480d2fddbab2e5d4170f5e24e23d2b0" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "anser: b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955372-i1Z3xjXe9MdKPtYfud9e", "proof_sha256": "ab6a9663c84a371aca12ad78effc8070a511b19b5e99eca616ec22f586d78c3e" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955419-M93f8RGafv2FPoNvgram", "proof_sha256": "713e5e81c2008edc59e938cda0ddffcd7b97600f03cd310c6bc458cf019b8280" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955467-JgG2pPESwFTwUfU5pH7T", "proof_sha256": "e80b28b4a27ec75c3fceb9d56685524ed0ed49ad35e7b885fa86de0266232c25" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955522-Vr1wmOpiuEjt4LDbixGS", "proof_sha256": "0552b1d1ce86fde8768bff4821c805ba0a1e38599233fc3d6626358cd12a747d" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955573-2lgEAi7s0whMEd9LxneD", "proof_sha256": "008b6c60cc17ee5eb5c945102c1852b356617957c7fb96fdfa7d80194744a2c3" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955619-lfnuMZuqS5ChYzr1r6YP", "proof_sha256": "b754dc560ca2ece4360f5164e1952372f62940459c5601afd84d2dfca47c33b0" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g}$", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955645-217UiR1LHK9CwhBIB0aT", "proof_sha256": "8af8b2e24e6039fd5979fd8692b618ae54058f043436490650d6945d4c16d628" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955663-CHPNuvt47WwMyTV0wT1z", "proof_sha256": "008500308ee035ff52c28850e1a4a9058e970f411a51fe9224235fa2fc2cb794" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955720-dC1x9ET1Y7TbCZnt9lIQ", "proof_sha256": "cf532e2c5d644306bfe6cd0781f7b4be8abea01efc86114b0f745e1b05a109fa" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955797-yCwNafHUzeMqPZVGmnXF", "proof_sha256": "2425c9c647cc890bfa8845454d4dd77bc9c6f4b4b430a9f77d89f4460563bb97" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955857-R86h1WjOnb41l1o46i0q", "proof_sha256": "5dba658289b7527087e8aaf04de58cb6b11a5158fbc4e6ab71af362318bb0d4c" }, { "model": "mistralai/mistral-small-2603", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955918-HKtZIwtpHpctdkUwQIk1", "proof_sha256": "7d5351995ab5cb5f9fd00f6a27f25609eb18c2711c935dba4d0579e058cec98e" }, { "model": "mistralai/mistral-small-2603", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792955969-LNur9WgKDLonA3vGeIAW", "proof_sha256": "91f30824db82d7978f682aa47e6b51cfae5a1527392f161550038e5d199429e0" }, { "model": "mistralai/mistral-small-2603", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956007-VG45LARGX2TZDdaWC3gF", "proof_sha256": "26de3fd507bded4f494146eac576117cf816fe5a78d707e7b8988bb3f062175e" }, { "model": "mistralai/mistral-small-2603", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956030-oGOJiLvV4nTkI79Mm3Xo", "proof_sha256": "6f1085533997a88ed812f357dffa7f2b05441f089fdafbbc42b061f02ed66682" }, { "model": "mistralai/mistral-small-2603", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956063-BiOydW39Ay3wuL8tud94", "proof_sha256": "930343e64abc29f2f9b2dc7bc16b304859cec3c2b420f775a6548456499e5add" }, { "model": "mistralai/mistral-small-2603", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792957634-XzxbkBSbMNtUbLxi8Rpa", "proof_sha256": "90b94172c4d4852280bdd09eec9b677a00e93a022260e37d0ccfcd74ca393034" }, { "model": "mistralai/mistral-small-2603", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792957664-GmSx0BA8fNBn4MBxjfRr", "proof_sha256": "21544b05c7e2c4eb01f31d654e678dca63cdc8ec444671ba12e0d3d6414c70fc" }, { "model": "mistralai/mistral-small-2603", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956127-EfV7HMgI0wu2K7oj1evF", "proof_sha256": "3a8082736265f6fedb3199bb83b227180dad988bc68f1974f251949402db47a3" }, { "model": "mistralai/mistral-small-2603", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956139-9ICdkxjPFdy2GuLE7pLe", "proof_sha256": "d120e17195ed8e913f204f31385ef2c239abf902adbfd04fe125c2ae14a49b2c" }, { "model": "mistralai/mistral-small-2603", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956152-aJnX4m3YymBrc5Oqrknl", "proof_sha256": "f70b016310601a5549f642ccf956efcbc3d7f4d81fbeb7cc9c8d140f1d87e8e7" }, { "model": "mistralai/mistral-small-2603", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956177-36ut44GVEck0Me2X3Y6L", "proof_sha256": "4765d2face9bdffce17f7056b64a0c63ffb5951580a0ef4574081db1a8e0c71c" }, { "model": "mistralai/mistral-small-2603", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956190-MZ9UsuNy52GQAbM8Xb5B", "proof_sha256": "99e89f51e006cd346aef7d8d068b3b09df9a4d57c481bdee667bad5b589ff752" }, { "model": "mistralai/mistral-small-2603", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956203-QZCJ8YmXV3jcGoq8jTgx", "proof_sha256": "44329088f911023d8b20ae2f24710a17abe191c83171c86f7b55ab02ce86d541" }, { "model": "mistralai/mistral-small-2603", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956215-mD0NX07JnWavMKcqjTJ2", "proof_sha256": "bfb68636ca1c6879e87062ff70f0408f2724b96143e8339f8394ba62ab09fb52" }, { "model": "mistralai/mistral-small-2603", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956226-d7J6CJTzYZ4DMB3WLOAL", "proof_sha256": "ccbb63b646ee0ecdf9c5f427e949155e78b3963d2fb27a1d223f15320b9e8476" }, { "model": "mistralai/mistral-small-2603", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956242-I0zyoQMgaMJxMvci89JL", "proof_sha256": "0fbb4b1bc2e1c680c585852340c1fb9cf4b936080bbb809d2b0990738c0001c9" }, { "model": "mistralai/mistral-small-2603", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956252-KtJctII5Q4xqF3apbJIR", "proof_sha256": "e25b41ed30c286e606e65999fc190400d9d36bcd0bf12306a004710516709f6c" }, { "model": "mistralai/mistral-small-2603", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956264-9b9CpIE0I8PhYnF98U8P", "proof_sha256": "9ee64247d6fcee3e37dfdd0480add1fca8661b414bbdec3295686f8b6e792b80" }, { "model": "mistralai/mistral-small-2603", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956274-QTnBHyGCrb3Wg9rkYn3f", "proof_sha256": "949ac7f60aa735c06537f053dd0c98fbe21f5b8f5445aca7b4dc7e56c6ba9a4c" }, { "model": "mistralai/mistral-small-2603", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956304-XMLlnJ125Uk8Pnc5juCT", "proof_sha256": "5ab71c30e9a08005602a69a420f0eebc25f0f5063e55818e01b71499a776bc83" }, { "model": "mistralai/mistral-small-2603", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956317-1kBeqbkVWkijivCpy1eV", "proof_sha256": "b9f053def2a20968c8e04076a30ad21447251b508755d87299ccbf46d47c8048" }, { "model": "mistralai/mistral-small-2603", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792957680-tuOlubLwW0uVqM2fwMWZ", "proof_sha256": "5fce2081b53f6068e78fd75c9866bb250e6162cf5c9e71d2a834030db736abdf" }, { "model": "mistralai/mistral-small-2603", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956363-O4LKD8Odls49m2cD3CQ7", "proof_sha256": "b906979d953a89d7c714761fa4ee56dd4fcea6d09ca3c68165aa7fed49bd49b2" }, { "model": "mistralai/mistral-small-2603", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792957693-f7GCzTK0Q2SXMEAFOWYz", "proof_sha256": "953d5e4731d8cd3e98d03181c6f7b5cdd593bc4cb879a2a07e3195b0e8ff074a" }, { "model": "mistralai/mistral-small-2603", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792957713-M2undjm1wKGgr48eHRhh", "proof_sha256": "926aa968caf415c2e6e4f367503cf61286a733ee714533a7ac160c314a127783" }, { "model": "mistralai/mistral-small-2603", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956430-UmGhenwBZekhgUuJUvKT", "proof_sha256": "fdfeea5423a960ba05ab395cd47e96d8fc45e963bc430a238e23b008ab0ccd06" }, { "model": "mistralai/mistral-small-2603", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c}$", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792957742-b7lgEm0pVR9BMqUYplB6", "proof_sha256": "f5b54877bd082a34be49b3f32e05f9ed6640c2e280f79cdd6c47e5bf35644d46" }, { "model": "mistralai/mistral-small-2603", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792956485-QIsIIfKQ23CP8O7xFRez", "proof_sha256": "3c2057b2764456b3a233567b61413fa47195a1dca02b76766d9ae35b3cb89745" }, { "model": "mistralai/mistral-small-2603", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e}$", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792957763-uIS46khf8bQUoYOUAuuR", "proof_sha256": "141f076c1f7b9b63d965ddda4ab4b747f42577b588a5f2a7d22d05c69ee0c73e" }, { "model": "mistralai/mistral-small-2603", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792957778-OIiJqo5389rQ7xZMUWl9", "proof_sha256": "9601c0137a77ecb8290080327f1bdc322899af377204f4f383ca57f2f1a0c0be" }, { "model": "mistralai/mistral-small-2603", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T200000Z", "generation_id": "gen-1792957799-aAnbEy2ZOqmKkwUPfv7V", "proof_sha256": "c756537ad48b9021615f9362ef624dfdb0ebde2ba2b5f7b8d712b4a453029f5d" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954256-pQO6nR0TA3D45AcBSRj0", "proof_sha256": "da3f91ac76c121b14d1fcaf66df0e134c23d505641372eb85308586a7637e413" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954269-PALB32uMXauSijBRwkTY", "proof_sha256": "8ef1c75ad1430108836770e5e0604c8782abd406d81fe8fc9cf84cefa21d5c59" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954291-kWU0ycwIB6GSB2iYyjMT", "proof_sha256": "d399c3ac5f15e04dc87763a4171a73a0c209b3ea2139099d7f6a593570d83c8c" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954307-2bHnPxub5QqJVtFwxB8Q", "proof_sha256": "3cf25444cc0b1611ebffac797ac8dc3453b4e149e34e9548810e1fe472ad6d22" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954326-LroM4oaWRp5L47I25d6O", "proof_sha256": "3b64a3bba189f7b19570a0f4e62e1cccdcdd903e0875173ec0ba6dd21510992e" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954359-TcchdRunkGAtJhyTkwU2", "proof_sha256": "9e54c584b0d5a308ef609f1b5bf588f72a354a3bb94e902d35a96cc0ed443f7a" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954370-aymJgwBrz0gKzGdq7udn", "proof_sha256": "73163ad9b1bf5d50b0d3d4ea284f675a0c11910d5a91b0a2734c855ca5c31173" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954381-fpNNakRPgGG8yDFZdDe4", "proof_sha256": "27d20ddc2cb8f1410be87a135c0a4c7480e23f6be3b133e2ab179d6a581e9988" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954391-61I05xpkgn6Rjr4n9Woe", "proof_sha256": "1887dae85cd27c4fec4fa24ed2cbe9fa4b83a66b067609946745d2be4a006762" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "GMICloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792954407-pa7h8NPYKiPUokBp9u0j", "proof_sha256": "2e8d052c868b50576c5bc66e541348162ec842ff8aeb8fd39996286c4a37144b" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954438-7Mc2VmLf0OfLki6hEwv8", "proof_sha256": "5d0de4b34079d6410acad48346879ea5ba37378350ba9fb18372fcc8fdaaf4a4" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954447-FVIvZAl9LmyKICeZHpem", "proof_sha256": "873b47820c8238fd79013406d8c39573509f86b017ac72aa8d4b520e71d5cf11" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954465-E9iMlO0I5MynmA5XK0Cn", "proof_sha256": "77a3925af16948556614711b3ea19b0a7b252d025527df6ff61f49bdcb679d5e" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954473-WiKF8QzJn7aLANxdgKR0", "proof_sha256": "21376c40ea5a8567ff4346d3dcb2d66a882d6ae26efa4fe422cf3296b2e46949" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954484-6kiO2bpXdKJZdbvcFLNK", "proof_sha256": "40c0262afac56d2bd6824325bbb570821d359b366a84aba42e9efe1356d051fd" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954503-kIfkwUP16gcRuSYf6uqa", "proof_sha256": "f11121acbedea56a1573b37e023f14aaed6d3b1fb3003ff168e1e880a173b9be" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "SiliconFlow", "run_id": "20261025T200000Z", "generation_id": "gen-1792954516-Yuh4czyVL6f3HoehWdNx", "proof_sha256": "86a94775e7815f8f1aeadc56ce2fa31b311e35be2ce04ddd358b3558336deee7" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954530-WCbcjxIcf96x4HFdSPID", "proof_sha256": "106319c875e066f922c1495aef1181aa26833e1e93fef690e7d0f5bdae1a9d37" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "SiliconFlow", "run_id": "20261025T200000Z", "generation_id": "gen-1792954550-5W8uVZnvetfPROuiYcj8", "proof_sha256": "32ca79ec578ea191d78e0086bd23235852ae12c3b40ba558995d3e4b4e411ae7" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954560-Y1J4IMoGaFhaTbj4JPzX", "proof_sha256": "da63b575a67b06e9a6e508f4029725c383c331c4d564543972d6c7c7b88f8234" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954570-hAQflZyMHUjqhlnyZegK", "proof_sha256": "eac2c91414f9b54b206c01435427e871b8ff75fa436b961d8cdea54a9e0c72ec" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954581-TipE13CtB16Ja9vtqzaC", "proof_sha256": "778fb0b206092bc1b8a95afe9ee86be4de1c641506c6faba59b606f2ed35c80a" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954600-FYROetkvTy1kjLmWo9K6", "proof_sha256": "50fdd8cc8814f066d81a1b85f3a55ebfb9c48a78c97ce219cfc97bd1c4e05bb0" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "SiliconFlow", "run_id": "20261025T200000Z", "generation_id": "gen-1792954619-6LVQEI1aAIgu2wJTYhm8", "proof_sha256": "48a91e9145fd61f5d767cf680065d2f73ac0fea49d79ca505ee5b3ba0e177577" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954635-DevvN0KOzmO4nc4q21Dj", "proof_sha256": "edcecf21135b8dc66d286c73cfce355a556ba1b90461d9d55664d9d92c09ce3a" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954652-EmYcHjzRrjBHxWn24JwP", "proof_sha256": "62aaeaecc7c6ec083ec162c1b8d0c9fe7f31d28a0e680f8a195c7abb794fbb75" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954669-S4l2GcmwCTSR4Sw4ADGl", "proof_sha256": "0840c82e0dd57aff5c5624d6d47cbc15fc633b19ec17bd5c4dbcb7fbdb6f2be9" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954689-IUH1tROfa765YqY7lqnO", "proof_sha256": "c246722b13d2152a522c0083d242237882648909c96ed78099072ad52b375ea4" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954706-x5ZpYk13S58QIjUOMeE8", "proof_sha256": "26520cd111fb87ad5efa417bc4c4a5be4473368f084f59b737eb61fdbe104795" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "SiliconFlow", "run_id": "20261025T200000Z", "generation_id": "gen-1792954729-aiiI8AF8VFPTbtxvEBeP", "proof_sha256": "554080c59780fa995ab23a85e2ad9e17adffa3d2d5c75b53c3ca557085298733" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954741-iNF0TqYTo116fGqOuqSw", "proof_sha256": "99370a4587c782da2fba37634047fba4c08a6bdf46ea035025c37989908dd8b7" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954759-sWd1qewmh5nPYXgJpsV8", "proof_sha256": "484cb0f150e94bf07e50a7a22d3fdfe92cb49a1702610c30d041bbd80a9e93fa" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954778-kPdQ0EqW0iUQ8GM2anxr", "proof_sha256": "bd21c622cc2c1d04cb38eff619b66b94c53a541adc3ba262f13136c707d593a9" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954793-nxxRk19iU4T9vdjpSSGf", "proof_sha256": "43737d97ae0ac9a121fb409504851ed0c9da1b529fb6c5651ba1928e1f630b33" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954814-u02M7ijGJ1JxFeZTiN4c", "proof_sha256": "0c6106592cf8602db4e01a9208bf271c72898a10cd41b8ee3d264f2ca0696ad6" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954838-R5QggagAGUT9tTdAuirj", "proof_sha256": "238c21ee7c9b980c735881cc29ddc74c0075d527955b4514e9fb74d72f01631e" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954862-oehH7EgxL9cWgzi2t4RC", "proof_sha256": "8d0906c91065178b0fa6b9ce6dd129df08f36ae93d5e62cd924d81c2ba5780a3" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954889-quJ7SLPI9tvS5rzb6RkT", "proof_sha256": "39c87df0710673e9eac2f04d962457ea735a9e253f7afd8052dd8d96ae3ab7d8" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954900-7w3WXgxviwLeBXG7gZWH", "proof_sha256": "b82468c554c986b9b3a9f10dbb1266ab3a990076da27e48fdd2e534dbe847a69" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954914-YXbS0ICX3MOMyTR42Zoe", "proof_sha256": "77bd1a2e98425788552f24c8cdc7e15b2c0eb09818357bbd1a301dc8122f42f1" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "SiliconFlow", "run_id": "20261025T200000Z", "generation_id": "gen-1792954925-cLqEnWss8qtL4ARC7qmF", "proof_sha256": "977ebea0bdad51bf22966d08db8c24cb70adcbdc9432970c2569f3e5a21d6108" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954937-fx7WL5wkvaTPlQvUYhc8", "proof_sha256": "08821fbbd3472a464c53ced8d7462c2342ef89fdeb20e09a2d26f64073a1ce04" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954950-vqt0DYKB5lO1xDzND3Lb", "proof_sha256": "34e921b32be20b28cb8e9986299b529f88987d5c77e34ae7610aba0b8bc39b5f" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954969-CiQi2JS6zW1y0WaabK0N", "proof_sha256": "f7cb5303f6cdde5a7965bdc07f68ba7c3fb5b8ca46c31e76363feacf0f8420c1" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954989-GvoauWRUfgQD0xHAU79d", "proof_sha256": "4f3c97a7ea1197d8491f8f83003adbe7ba17048c6ec0432c0cbe00c38c376d4b" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954997-wruJXdwXRCKngU4szxMl", "proof_sha256": "e1582db8253b858fd6b03187c4630b0283282ef155aba8d5099dd25ed5cd4b90" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "SiliconFlow", "run_id": "20261025T200000Z", "generation_id": "gen-1792955006-IBNoHEMLw2lh0YqcQL5L", "proof_sha256": "21d1a35eb579f12e60eb3a19e7d9a722dd26cc3753b1c141b221a2c38354bf6d" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955014-NYDdUXFxT7vg6IilP3qz", "proof_sha256": "6755c2a4d87fa2489d02e29318756791f3b9ad595723e2e8f6ffa77e08f85e1a" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "SiliconFlow", "run_id": "20261025T200000Z", "generation_id": "gen-1792955032-EdVZ7OfARUDiv2bgrIls", "proof_sha256": "30c66f9da7883a3bb39d3b6222c5d35dea260754e406f73177b4bd0f3ab225ed" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955067-3w8NsCkd52q3VzMrpMpI", "proof_sha256": "bfbdbb04c2c79fcf2c9d0c6347a4133876d176cc55ce7415b1536c120069dd6d" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955078-ApFeJ3Oi7d5MRBBCLjBN", "proof_sha256": "171bdba045c90f944709fa15cd0fc7c7457a34451e05d53ccb7c332cdc247b48" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955097-dUsqqDfSVoTwlmeh4RAP", "proof_sha256": "98bf4c9630bd8704eec985b03b48bca35299ba539e510c9d2e00e40b472d4467" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955117-XAmLSXwEbVSai9uMuVOE", "proof_sha256": "8596fd78c7640b72c421b6a2a9a19f1b2d43f3174acba1fbe0f12652a73a2ddf" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955127-qs0f7Kyi70oxHkzKHnOS", "proof_sha256": "976a0ba36740918956c8cfa0de143c30c6b9c3b9e6e70a0018c499e9cf5979e5" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955150-Ur0gOlitD2GHwiYMVyOp", "proof_sha256": "b6dd9b80b28f06f238ce94cc6f25b60890893a15a9ad6f58c4a7edbcef5416ea" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955168-iM3DJWLlVbmJVtPEP0aE", "proof_sha256": "08a20802b40410370b4f55d3a213cd71afef354b47d30847458b70af9765d7f3" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "GMICloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792955184-YbEw9wJSeWdbaIJcxRku", "proof_sha256": "0176b8de170dcf0335bb6c6ea77846e367dcb9bba209664b266e5187d13ce45a" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955211-psdzKBWQzfh5S5jSsDP0", "proof_sha256": "500bd7bf502a27c819f2bd65c48f571fb6c7c3b282f18b0acf0f24293f95664a" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955220-bljL9IlTbSTt1dBspGvR", "proof_sha256": "6e557057b2a073d3955530deeb1c5e4247718daa45d5a1a82d951c729ce4d7ed" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955229-nQTfIAGOI0oXy37trx0u", "proof_sha256": "6cfdf077052fd005ce51f2174df6d8d7a9309bc5a764e3761238cd67a6d9225e" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955236-SYJHX3co176bV87ickeZ", "proof_sha256": "53cd30a7608dad620963bb869972e492379752178b870a37b1028471a4df1ea4" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955246-uVajBQoN6Rrz0qYYHKiM", "proof_sha256": "4c9fe59f6b627207334d6a17f616d4703f2c0a318d49810d5565f62bbabd5f02" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955253-2dKyEt9fMPF1M1Grtx2H", "proof_sha256": "9293808bc8fd849c9a6ee9deebe9bb438edcac9d5a31777c13e4a35bfa33d590" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955263-QOhVUcOa4TemvddUm4CE", "proof_sha256": "adde203d5ca599d8031eb049577c2d775500d62c46cb37f23edf0538db834de7" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955272-Was4oQOfSffLkxbMu09d", "proof_sha256": "4809d1fd4f9c5b8f4303532000fe835c2da006b7dd5dcb9a3d3b9927e3582137" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955281-pEOu4DKNeJJ8b9rpLzjJ", "proof_sha256": "4bc7ef2185ac495adbf80ec14ed227891e8d28c84335b332c3818762403b28c2" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955289-vjJ2jbCUqXRNwX6Bsqc6", "proof_sha256": "aaec693afd45d128cd8a94363658c900496eeed42e1c419d668648ddef8320fd" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955306-wC45i9KuvSGsxO8jgQp4", "proof_sha256": "d2616ff5a592b24a4d2d85c4c23a092d6876b5c8e9cfc5ad7d370fa89476da4c" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955360-LAfNVeQqP1WoNLyBFSva", "proof_sha256": "cd41e958172e976a6370733f2e19c786f4eae220830b7d55db56d3e21e53dad6" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955398-pR0lI2ivFGsuRL8XkCT7", "proof_sha256": "7d9566dd819ac80f2838f5febe3eb9317bfae7a059d769dc753ca8ec5a3c06e8" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955436-V4QbLiMYwU7NXTnK9J8z", "proof_sha256": "68bcbf781644528c54a20ad867020da4814583460d007d5753d58c5b485a4eb9" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955475-7TQtS9Ku3lkiSRqitR7K", "proof_sha256": "465398c512fa1bcaf9821e0f3aa006cd9bfe1b2024faf4da291c5a6331b3efa4" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955528-D2rSEtuNK5aFRmFukGZu", "proof_sha256": "ecbddc8f74dafa47e38f9f37dfb6c16150ee3b74ede59b96e4d143f62f6c8cda" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955536-CbIVc19AgiT0J3fbCdA9", "proof_sha256": "76cdfba5cf7c4ea02f90949c3332d2073097836fe2414be00f577d37cf60fe57" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955577-obfv1g4vIF60wEwXEOXF", "proof_sha256": "9fde6e811f63a029b9fb5b235d934c17095ddfa88e1c2ee89776a545635b953c" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "SiliconFlow", "run_id": "20261025T200000Z", "generation_id": "gen-1792955617-ZJszmIljeVFXyrswK4F4", "proof_sha256": "a261555304109da5a5ff8fbf145e7e66756b18d56243d4e927d38d34197480a8" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955658-ke9lcsNLu7gUBIqT6Dhw", "proof_sha256": "d76338a8822c1119fa7feee3c05900952958610cde7e908d5ca1d4fe5a4af294" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955701-W0DWQaA2S20wLeWKZYla", "proof_sha256": "a07f164170bfc7f6148f8c2fcf1352217fac3988a1a7a4cfc560dbddcf613fb7" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "SiliconFlow", "run_id": "20261025T200000Z", "generation_id": "gen-1792955741-b6j16i0LXBiU0iRwuyk5", "proof_sha256": "f568ed0c61395360aee8120800c50a2d72ca6a754b17d2ff3684c12f7f690a4b" }, { "model": "deepseek/deepseek-v4-flash-vision-exp", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "SiliconFlow", "run_id": "20261025T200000Z", "generation_id": "gen-1792955797-3ymOCcN5jboJoCw7zPep", "proof_sha256": "520d89af7e878c2e814267d6d4115ae7e302b2811e30c7db574220c3a2bf6f89" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792954256-G1ZhuDKd03GXbaBnX878", "proof_sha256": "a7ca70dc34975ae6c7237db25223c3b4868a6f5da6787c2b6b4f0d2cce363cc9" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "- diver", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792954276-PsW2CyWzyqOPc4KgMrCm", "proof_sha256": "4a83310a36c8477425bf39bec6b6237007fe027d1aeb009427236a1bcf3b1fea" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792954324-kzA2GIeOYLnbtL4aKD2R", "proof_sha256": "d3dba5c66d9a44d8a0df64ff778db7b448bd0a27db67649df95d3647860b6330" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "-\\frac{1 + t}{2", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792954374-dYWAq37gj7tLm1YXRlfK", "proof_sha256": "957a5655294dec6d732d9598ef429d16ab44e06b956ddedede09d474ba2d3bbc" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "so with $ r(x) =", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792954423-PKg967V9EU36cJmTwqOh", "proof_sha256": "e94fdba1c73262778d3a970922d3c7ca1bb38e4568e822ac00a246975839f978" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792954472-6sTmrRuzWYFUgC8jzdVH", "proof_sha256": "6f9ecbf04a31ee82eb8d67657b9235aac48a36023fe6be08e9312e57e83e81f6" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792954527-1ksOsS8cao3TiM0uezYx", "proof_sha256": "52aa111b00f6935c9e78fcfe1dfac27d78e086f8e680a60df88fd93598959b1c" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "pb = \\frac{b^2}{2\\mu0} = \\frac", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792954558-scP3QziqBVcwnat1AJ4a", "proof_sha256": "f1f4c485a93702b1569bd35b079c7b656a1eb9f4b7c36cd574f1370553094b8b" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792954606-b0OY9UmqB4gj2ukW9SZ9", "proof_sha256": "461969cf9da656090ce30bef8af4e8f218ac14f2fc560b6eb196f55935cd87b5" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "lets check numerically: $ t = \\sqrt[3", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792954644-yTCiwk6WssrxGN8DFenf", "proof_sha256": "036d4e8e4b99adf56a2d9fee04597d4bb2b9bc19bc0215d0a0ea0c073694942e" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792954704-xomSV2YQUeN1sSRtkpku", "proof_sha256": "54d20ff63e500667c6fc8255a10e4e4c71a75faae91775a1847edcc4fdb7f255" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "j. 89,900 and 21,500", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792954741-5hFfQKJ6JGGhDYGmQGib", "proof_sha256": "46e6b95d698fd05c0765ce920772d7f13259ae0c410ebcd5efea84058c595d33" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792954799-moRHvMc5xf3fsiGkCqiX", "proof_sha256": "d7765a2725e8af829e264abdb92dd80cd719c040a2dd98e144fbe8ae2b22dd04" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792954810-tmkXmIe8HdcyHztNXNuk", "proof_sha256": "a4b8a110e42a44326f5c4a39ebe04fb5bc8a24cdef3cfa407027b1069eeb2213" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792954847-LBdQber8k1i3Y3Zl0lE2", "proof_sha256": "5a192df111eb7e98259e1a94e558ea5c06cd2427a1b4db677df5d27ead575004" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792954895-IuMmTBOqnd00YVClbsNa", "proof_sha256": "c9966efd5a39a86c0bb4f6de6638c038e8caf95c835e2b9be59b62f064664549" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "$$", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792954935-xEvJjyMStAXRVHnaBH83", "proof_sha256": "e74fdc2d620df5c18564ef8ce1197d56bba0c602f21a58cf7025dc556a9e7bae" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792954982-yiULGyy6sKxqSeajAhFe", "proof_sha256": "55f73722b6f49b061f3aeed0c3e790b37dead1d7b9f04ed3395ae28f80f67642" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792955013-IAG8kRkjo7xL9Kl5P2Az", "proof_sha256": "6d10dcfe4499b10bd736d902ce64065800897676d141f7e276073a0d1fc3d6ab" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792955048-RiwYvAlte3HM7FiTZa14", "proof_sha256": "9d1b9ba79d45eb5f958863916ee278b63bd49c01febd9aa9bc1cb7fb08696f45" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792955064-WxVP365coZtZzVH8IQiM", "proof_sha256": "76500c7ae772f14ab7e7bf414782e8ac00679af95bef7fd5537bb8fdef799fdd" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "lets use generating function: $(1 + t", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792955100-Vp45kmExk525g9sHuG9M", "proof_sha256": "3a6ddca0bcf875c15971682fa2fcdeb7c7d1878da7d0f78396003c245ff1c348" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "\\delta g^\\circ{\\text{rxn}} = [-1058000 + 314.8 t] - 3[-111800 + 87.5", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792955152-m33GuALyxnPBxNnxRQ1C", "proof_sha256": "214071e7e30f57dcc3e3302820fc1144a8f482bd7653a60848565f1dcdf9dcba" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792955200-XNyysCItAuaF6Ne5FQ3r", "proof_sha256": "55156f898dcbe028153ce56c916c20009014033187960bcb185348458ae19a06" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "\\( \\deltat = \\delta / pr^{1/3} \\", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792955237-XjSxBcQJXjkjHqIpdPIK", "proof_sha256": "6d59fe1f2b9c7d60030d1b348266060089ad1a75a1986141aa1bef8026a68e0c" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792955287-zrCQY9QaOl8y6FZWb3HD", "proof_sha256": "f2cbd1f1a7b443f8c1904190d9eed67542f15da6fcdd26306f7bec2d7e6ffe38" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792955311-bTQr7lEFLOrskqNY2Rvw", "proof_sha256": "5718b01ec7ed459c08df577ecb4a9d817f5468c009b6f3d161d0b19c7d167428" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "- $\\thetai = 45^\\circ$, $\\cos \\thetai = \\frac{\\sqrt{2", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792955386-a9gMvUJlI6I3s7Fvfw2C", "proof_sha256": "97e28bce80148290ef1019fe1fb4822409eda8766f52c90dc1a403f351475a30" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "v1 = \\frac{nrt}{p} = \\frac", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792955479-79k3YNnAvpP2UXq1ckQs", "proof_sha256": "39d0e2713aefb346306e3a236e63ae54f47842dc9730be74bea9edb4a5ac271c" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "maybe the given $\\sigmaa$ is in $\\mathrm{cm^{-1}}$, but written as $19.2 \\times 10^2\\ \\mathrm{m^{-1}}$ — but maybe its $19.2 \\times 10^2\\ \\mathrm{cm^{-1}}$? that would be $1920\\ \\mathrm{cm^{-1}}$, then $r = 1920 \\times 2 \\times 10^{12} = 3.84 \\times 10^{15}$ — not in options", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792955559-m1QGLB6z4uE4BQzbdajz", "proof_sha256": "232660b23c2cfb155c10e3820225f03d3e9bca9e1257a79317b39bc08c18617e" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "\\tau(r) = \\tauw \\cdot \\frac{r}{r} = 16.907 \\cdot \\frac{0.06}{0.15} = 16.907 \\cdot 0.4 = 6.7628 \\", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792955631-AT1WlMYHTzLaywClsmpY", "proof_sha256": "947869e0472bbdb9fe2826da31c8c5d6fce13fc172ab1fab3c60c499ab1e9383" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "maybe the unit weight of sand is dry unit weight? but medium", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792955797-TpsENbrqypmvawEzuJrs", "proof_sha256": "790bbb3e47923be01ab4e154d2322e08f91e56f84b3aedc6a4f9a41c78bfa9eb" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792955855-NA2vEHVIIEZx6rBWBNzS", "proof_sha256": "ea5e6eeddb6485978619cbcf91d4188272d0c3eb8896ae0dc85980bb164a25e5" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "option e: 0.09–1.00 → re = 4", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792955942-5CUsf8ePdSXfOMZcCvrk", "proof_sha256": "d401db326c57f6345209025da63a4adad2227a314d9ef37efd95e60633fae1b9" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956012-KTzPvQk5WRs6xRpazj52", "proof_sha256": "9d6b0c5fdf26c1987cbd94ff980de8ab0e5b1bae15c69e0ad9e2823bd16b7356" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "$\\mu = \\frac{18/5}{627/80} \\times 10^{-2} = \\frac{18}{5} \\cdot \\frac{80}{627} \\times 10^{-2} = \\frac{18 \\cdot 16}{627} \\times 10^{-", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956058-oZzbxWKNHcuwDKsZnB60", "proof_sha256": "650b37eadfd39be94f152553a6598e3ac3e5e43a4839f6266cd9d965779e9f54" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "answer: a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956125-oy3JVFIDmKCoRcCvs4Ir", "proof_sha256": "06f0889eb8646326a751fe9cd2dcbc5f98a2d1ab9b102f367785a2b6ea48ac4b" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956142-ZOuWhYuaVWkXLPjY0GQ9", "proof_sha256": "ac5caa2ea70a3d199b544bc19a44c5d44ec64911ae332160483754ce860be6b0" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956159-TeS9Xo9ykAxUNT1v5Kar", "proof_sha256": "61d7b9a1c35e4351f6b32b6901474698473d2da0e1a26f46c21c3a15e8c45c83" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956183-EpaStfTm69qTMcsN2PbK", "proof_sha256": "a210d0998ae7bb5f6d8f364705f844b74e1f82ab9f23f7e176ec54ff9bbdf86d" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956210-dQNAm05hEeP2Kx73q7Fl", "proof_sha256": "2b141a9e68c71deb3f9aefe2d5241dad85327cfcfa5915bc0cfd0d8cbacc22f0" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956236-DYsBltNz0Bo7C3HbclZX", "proof_sha256": "9c91a6ed2fd917d2787231c9aeaeb25dbee9df26154b3f43bbcce49afa80e098" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956259-KjaNTcdJ83ZSK5zTlq9Z", "proof_sha256": "f1f066b7b27a76ad3903c977624f00c3e66d627f2ba4bffc3e65db86a879e503" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "wait — maybe the 3.4 mg is correct, but the enzyme has multiple active", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956280-LP9ijjaA9dYHduPX2yhr", "proof_sha256": "3d38c4fcf2e1b12ab489e4098b20c74b499152b400dba928e54fe1e06c06c00a" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956327-h7SogUDmpBT03AjBRal4", "proof_sha256": "a145ec53d1f1013b2f9c40fd8cac285ae7ea32ff541777b0b71d67557e2cc725" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956341-K0KEu5xhBdQRDOhfQqmg", "proof_sha256": "fa63dfe96c451c8a5cc027b66cabf5ec91e9260fd497ab27557264084dd6de68" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956385-cSX34IJxQMxRPnrN22K4", "proof_sha256": "14604454c2fc106c4c01317f2974cf7eee28b636282a10fbdfc4a5e37642cef0" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "wait: she works 40 years (25 to 64 inclusive? age 25 at start of year 1, works 40 years → retires at 65, so last working year is", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956397-A0xweCEs0tOLJORYrsOC", "proof_sha256": "00d5f4b5bdf4f6e36e24df0d91e07e1eaf350d78f13c2d89dcc0585b4804c093" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956453-ITycDWJnQZg6ndnsWwG1", "proof_sha256": "149a69f1394a172b5086641ee0caf911b3e5c84199211972daf790920d770e90" }, { "model": "qwen/qwen3-coder-next", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956482-XxZs1xKVKoKcWofRWZ0c", "proof_sha256": "e78dea9c562888c9a6af7b973a6890a7feec8f0b25d34da7728a28eb61f64ebe" }, { "model": "qwen/qwen3-coder-next", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956494-mid74P8Zfuygm32AehOQ", "proof_sha256": "6351943fd9c3012507050e39cc67d5226424b150860cd7ef4583e264292772e9" }, { "model": "qwen/qwen3-coder-next", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956536-2SOFshAMNsBCMDj2QlQC", "proof_sha256": "b25468db50effcd512e737dbf4fa43f525821ecc34e4177b96a69e6ee434ede7" }, { "model": "qwen/qwen3-coder-next", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956560-2Dq2gxR7wQbXJWyQQkGI", "proof_sha256": "14b30e0c88f3d67cb29b4a2281a144b77bea2f5f3a81c75af703deba1d5e1828" }, { "model": "qwen/qwen3-coder-next", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956575-VVDEbnWvU9qShcdY6BjW", "proof_sha256": "ac2362fdd5a9353988fd66c0064e6f265d049a879506d0c9bc92525ba9f6af87" }, { "model": "qwen/qwen3-coder-next", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956601-HnaLO2akTEanmnQoNyur", "proof_sha256": "dbada4d3ff4ca87d1a992e1b5e5439f8b41448fd25fe7fc7f3cc223aa3224b7a" }, { "model": "qwen/qwen3-coder-next", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956615-EBMVpRh8s00B60s0INnS", "proof_sha256": "c94c21715170e677650922b1ffd1eb63ad5f3b5160a7343317874af72397d853" }, { "model": "qwen/qwen3-coder-next", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956625-OOSH68Cix6BtMxFoFCQX", "proof_sha256": "07573cfd2da5a1441c483565d0f301fe2d564b9d547da35eff641b176c93ef33" }, { "model": "qwen/qwen3-coder-next", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956635-U2KrZqxLwtp7Yl8prvEg", "proof_sha256": "74b74b734a529fffb7876b898847cdbf884b235521562f120aa455e1b3e525be" }, { "model": "qwen/qwen3-coder-next", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956648-lFrfejpj4tzY5ZnPbVKk", "proof_sha256": "d582c6f341524542a1d71e2365dacec8ad22f9d5b8932bde6e708f0fbb9967c7" }, { "model": "qwen/qwen3-coder-next", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956661-LexQdcbuLiCDD3uLQFPK", "proof_sha256": "59c005d783ac268897d51178ab3c3fd3f8b7577e366e9f9b39526bfcc638d838" }, { "model": "qwen/qwen3-coder-next", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956673-t2nxFC4BhXCGnHvogJdq", "proof_sha256": "0168ca900aae3fe2ded8a8251bcb6466b43a2d13faf83df9252be1816c1992c1" }, { "model": "qwen/qwen3-coder-next", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956684-I4k7K8GSKm0FhhkkX1hq", "proof_sha256": "c2fb72da5e31a3843989ac8d6c4bd12935466064cefd9b3f77f10b872498072f" }, { "model": "qwen/qwen3-coder-next", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956703-wsBw9IO7GkH9jqMJ6JFy", "proof_sha256": "9b714baac635a5f4dd2dbf82414a11ac792acf1bd5ae82ad53f1a7ed24e68b6b" }, { "model": "qwen/qwen3-coder-next", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956713-yvtTR66ityMcEx0LLoN1", "proof_sha256": "429882019ee04a3967d5e92cc885dfdd2de3188856b663f26eae3f120c12f5d6" }, { "model": "qwen/qwen3-coder-next", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956729-rtb4raewogRRzWeuOCNr", "proof_sha256": "fa349dddeffec5add26574935c8e4f9f75d864b053768ce411c55c500209a274" }, { "model": "qwen/qwen3-coder-next", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956740-v5alz9nJ4Vr446joBAKa", "proof_sha256": "99e76697b1eeff6baabe85f5d8cdfcb70ce74d005dcd2b9d2e0cdf2c9eed31d5" }, { "model": "qwen/qwen3-coder-next", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956749-Uj2J6ZvJ00owx62LdsCZ", "proof_sha256": "aa55a0adc1279275d0969921c4b3079f85f4e2d328025ace4e9bf99f76701c9b" }, { "model": "qwen/qwen3-coder-next", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956761-Oe2OUGXu51VHeI7LArsl", "proof_sha256": "1cbc7902a0c34be8c592eea36f6ea56281a86243e9f6f847b19045b4595e3a42" }, { "model": "qwen/qwen3-coder-next", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956775-oyPv0OuXp8GmD9NvfQ96", "proof_sha256": "91a8a76f49fb78473b7179fa3be34cca0a73e95a7eb50620ef22698180d0595e" }, { "model": "qwen/qwen3-coder-next", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956785-yFfNkZDvCtwyp7rYaChK", "proof_sha256": "6744870094e08c6510f0b0241ed78f15eac7be67ce819c7b95ccbf4b68c0218b" }, { "model": "qwen/qwen3-coder-next", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956797-4cJWrg6v9gdM3WoAQ6oa", "proof_sha256": "c29db044b40c21cb4b0aec2f28ba7298b5f515b6d56e5a6fda3250cada501c72" }, { "model": "qwen/qwen3-coder-next", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956817-harL49KxwbhBaWh2188d", "proof_sha256": "54c9a0b5a6db2491dffc98a29a3a5bc0a76cbaa10d7a4385199ee6b79bdbccbe" }, { "model": "qwen/qwen3-coder-next", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956865-gUp9n16pLx3iVless4yX", "proof_sha256": "309b5c06e78e5b4fad51f7ca2223063231ea44c83aa814344da55781f056fd61" }, { "model": "qwen/qwen3-coder-next", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "1.6555² = (1.655 + 0.0005)² ≈ 2.739025 + 2×1.655×0.0005 + ... ≈ 2.739025 +", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956885-54ARBMRJ8q8SHT5HJCm7", "proof_sha256": "00ca90e429f6f7e872c227f9c2fe0a9cd169e836e2464dd9e85fd7aae09f9155" }, { "model": "qwen/qwen3-coder-next", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956931-E1md5owxS5YPAzWwfYxR", "proof_sha256": "7233192aea5bb4968c6f784c15b51462f2d3c8fe1a6bd3d60a095a5d33a4d589" }, { "model": "qwen/qwen3-coder-next", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956964-Yud9IbdUA0DHFErtVun9", "proof_sha256": "46ecd73f5ea0ffd6ce3d8a4640dce91f7674f49552c788ad307ba29fc2a8538b" }, { "model": "qwen/qwen3-coder-next", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956984-nymJLZXEIJwnjv4Pmqae", "proof_sha256": "f3d1f0c2175929aee306c2e1c615f29060c58a04c4cd7a31524839c94c95058f" }, { "model": "qwen/qwen3-coder-next", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792957011-3Z4vk5lr08TzPV3Yiccg", "proof_sha256": "66f1e10a4e08bbd040f36a1760e7ac378d017bab26570a31643d694b0163d9fc" }, { "model": "qwen/qwen3-coder-next", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792957022-iZ531qUShKvnNo8JPPHS", "proof_sha256": "bd24bdcdf9b8f4608943f1761c2efea1bd0da8c358835e8738caf20d7efa0b57" }, { "model": "qwen/qwen3-coder-next", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792957034-rwCaoP1TYsK6YqRhbLaM", "proof_sha256": "dd9369e5e748ec060de6bc8e0ba055799cf082abb9cb92afeb84d616787ad7d0" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954256-kpGVaWvF8iMsCYPMYi49", "proof_sha256": "93d34ddd9a79d3fd952dac4b625727e0526840bba7f0151873148caf4c8addfc" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954274-peREuwgc57rV8a0Tk0Jn", "proof_sha256": "ba3c67067e6f3e71ea43bcd4c3ac9e665f4190cbe3b92206e7e24937d8e8b072" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954291-zuLDs9DP0lO9yJGozcAI", "proof_sha256": "fb3e667de38a944e9a756cee70c93b12ba3ea864fc561545bb6239e3cdffaf1f" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954308-8amDI89Omk8nNyiESbf6", "proof_sha256": "bf797bcf851be4db2be53cc9a2770ac577459bfa9d5fe182f9856bf5edb30a7d" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954324-k1GuxBSXjhS7h1mWdmE6", "proof_sha256": "b3d8ce5aa8078ceebdc38add4e55ba4f61f6c35b45b7c9d543bb7867daf9d639" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954342-qmxkIJNS7KeFNLr7hdxT", "proof_sha256": "6a350b9392c166317da25ce500078d06246d61e9fa307a686be3ddbd84cc515e" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954359-TYTNxFO4jaPSNnpwvzjX", "proof_sha256": "23ff8cdf15257c03f0eaba415ee327b7c385e819e738f1eeb3ecf2eca3ab71d5" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954371-uuPUCHM8mhp3Tp2YDdGQ", "proof_sha256": "5a5615e9af375fd241a25d753e3cfa8e5460f29a785811a9fe22a072595e39eb" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954388-XYD8UxhvZf3gfkKCLteu", "proof_sha256": "901de4e93d1b3cba4365e872e3376688cddd3997b9a327298b3dc4513b2d5b89" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954405-D72tS9PdUltelvtImQm0", "proof_sha256": "b9b588a266428078b16d38a801e618aa98d618cee1d933f157e6b757ddd2cfb9" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954422-KxCfFOUinMnCeuYIfhVp", "proof_sha256": "7aae5ff5ffb13e3eb801efc4e1119564598b58ee9ede73e5cad81240fbf3f7b0" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954438-d2kDKlFYdv2TnUjbzDx0", "proof_sha256": "dbb321dd02b9a9ad19f9ec9fa22168e96726bb8fdb482d2a9406694b5fc4a20d" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954456-YGJw4XQQsgmkvl7kIkEJ", "proof_sha256": "ca22e2d405dcf4b274eff2041e0648cc3488f07e6abb5055cea757bf0915e0de" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "thus \\(p2(x) = x^2 - \\frac{1/9}{1} \\cdot 1 - \\frac{5/144}{7/144} (x - 1/4) = x^2 - \\frac{1}{9} - \\frac{5}{7", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954466-h5MdFdQVAWdiqDhitWAw", "proof_sha256": "becd04339e9b945219002cc16cd43fe716d5248752231ce595069bdaf72ce7f5" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954482-EEtWCXZjtDDND9yPzEFb", "proof_sha256": "0938e3db5d372ffa4b5f609cea75fec2661518e9855a39b91d8b955adeca31b8" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954499-Le84wer5jhHzOHjBLuEX", "proof_sha256": "6b1b957440034b2217cb788a028e827356f30dcbc0daa2476a751546985a406e" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954517-TdROJkRCHgJBX1gFPtiE", "proof_sha256": "7d369186e82523baccb5002a32c7134411ffad72df33d3af708f97ec49844a59" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954534-xbVdk1CdHRZDVgPKzucY", "proof_sha256": "de6c2d0a67307837b95416b6e8ed7cf48458f514fb5a860a140c298e214b7eaa" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954551-CvODvJkGQmPJXsyRHJKo", "proof_sha256": "bfa308f6b2cdc75a48a218c2459ea648a8cfdaa464774781ceb863f139393541" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954560-7lkUhW4NPyvUoH1dVSNB", "proof_sha256": "d1973e3fc76b21162996e2f9b84b8e4ea681b65169575cdb949504567e49697a" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954570-n5PIYC7s92J1oZ3zct8M", "proof_sha256": "0e3ec0017982e3b096b6807f4526ed5922562921b0ba5e92a5a5f9e5084755db" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954581-MjDo631WNPJHqqJEHTc3", "proof_sha256": "28700df888cfc1bd9ca3494ff7bec9316c475aeccf93d482b2d5bfeabdc3b784" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954598-Nzkqx3WjhJHsqRz4LYtk", "proof_sha256": "4af178f3cffd9d644a2728f7c63a5350d4d321e3154b497a761b554a51c2fc7d" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954616-pXOl5UWvR7jOpnopWj3W", "proof_sha256": "8f092fd1e024fef7c4d8e5d2ff3656c72e11ea6c1278ca306084801773eb76a0" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954634-T2GIzO36HV92bv3oQL2d", "proof_sha256": "e8787a43ccf9ac6446b2f544426f6bbe4da9fce504b0671cb8454ab0c6b78fbf" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954651-oLBaOxQoIuRPHB17xTGC", "proof_sha256": "5a13017c9c3235032b71c3e5944fd75fe01038f06f7eebc1c3bd365652b32464" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954669-enz7IUz5NDHkDcdCF1vq", "proof_sha256": "d95e0640d2944f048582f0a6389fef933ffbdc3cf21f593b51af24f250e867f7" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954687-Rf4eOj4n97rEBZLWtPcz", "proof_sha256": "737c0bf186e0053c1fc8357f872aeeed79d4672a99e27d1022d61f1b10a3e248" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954704-iGJbNUzhiuWGodpDGuPY", "proof_sha256": "22b557b7238521a132c16851129017479f52644c0859cfa7383348fb8a3dab09" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954721-N67Udk1SL7icNv2NDhVO", "proof_sha256": "499f8398b02854f8c1bc10b2246d3c79e848bce669e9f711e5d4ce161cbb5f1d" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954739-DSBjCnULhzWUXr7SZZfm", "proof_sha256": "3c9a1f85b10e076bd983449be211e8a8b4f7d9fbaf8f5ba2108951782c65ca33" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954756-Y0nxIa4gV7ILy1PbeJw0", "proof_sha256": "01a9fae6c552105d67f557d6dc89c6284a7aaaadbc00fa5178cd562a9121eaf6" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954774-gLivOXmd6d3M0lTOd9ez", "proof_sha256": "887f498773fbd21d5f688503781428271e96eb1b32b43f2043636ab747ffc8bf" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954791-MhCvabrDOIjioF6N9Zid", "proof_sha256": "df25ef551443952c36717cbfade5b2095fe7177c19269469d8956f1b5758c728" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954809-olXS6pF3DQWbqxZ610Jz", "proof_sha256": "79f04e9f51d933dea5d74592104c64dd551d1d36dd0ad8909a37242682ab8366" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954827-EzvSYnExvi0ltZBj6ppj", "proof_sha256": "6c5d69e54ef0c8a14a6ca6d6cdf13522c66c794b873897ed129ee4937dcf393f" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954843-3sPEbkOQJ1Kc5VyLLxjE", "proof_sha256": "abdeda73a1e8d22cf14b39845984fe05cde83be1b0ba4e0255406ec7e4816cd0" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954862-X00gPX04GJhZgIVDcMEv", "proof_sha256": "2bc41e5927f9a1ded8b245c19feddb605067755165607679db289813b53cb362" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954879-mADAn9aLXOebPkXtTwNx", "proof_sha256": "636b4ae9b98cf520589f53624bd1ccf732eff69a82cf1202e2d74f665fd33a30" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954898-InMNEJuiQq4a5zfnCXWd", "proof_sha256": "7172af9c44d8521c68995f7da43dd1197dff51c3bc5b5bd22ec519b150a50f6d" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954915-IO6crN9STu2WfXWNZuF2", "proof_sha256": "fbdea749ff3c2ee441a0e1722579de5b8160812ac2c15d05c2157e77ecb24e7e" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954925-0H8DauGlHoOKs7tKlxxY", "proof_sha256": "11abab0e30225bf5aa30f9e3350b4133760b7936971e5dd4d344cb95b7fb871c" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954944-KP04EB3DiTPh7MXA5s0R", "proof_sha256": "8e9cc708b7f1ac1eef0ddc128d325b1b7481661bde6b910e8e7d48edce73e782" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954956-kUZ2fwJ8JvVSWywlXOH2", "proof_sha256": "f00c6f2c9dc560dfc3c7d3191dbd7615e86cfc26d5e16beaee2e8d877f2bda4b" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954973-qwK7dP97HooLaZmC4WSf", "proof_sha256": "d078011714146bb82c11c5e5d908b635277fbe0127484ab14516725573d7937d" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954984-rC78uuQ4f0tjOUJ1u3Xf", "proof_sha256": "df366a58e027dfb6850878a9ee29e0c619e7475ca08d904baa9a2a3c74564c0e" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792955001-8d2xhYWDXXjvvE82oo8W", "proof_sha256": "58e371aa9309bd9f2948ce83dba7012f2dc84478f33ca2600fc835ced392289e" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792955018-CR3sl9gF8hJETcV01OKQ", "proof_sha256": "ac63ad903e880c22398c81f4619d04be010dcc691786f406c8bc0daa2b002638" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792955034-z9UDVhdJTVNVzyEAqVGe", "proof_sha256": "7587431283b99e290b0dc07bd080b2c267c4733a2eaadf12e13e4fd4b98807a5" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792955061-gIinAdzatY0V5eOYViQg", "proof_sha256": "9c12cc6ea6e5a281544de8934c9f4fef331688afe4bbdf8c7e31fd89f5cf2c0f" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792955073-5Zgi9VpniiY29jyrZzuj", "proof_sha256": "98b56dab97a7b8087bf1bd52294b74fd2aa60c7b19ffcfee6e4301671152d8ff" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792955090-nfb36MxfTIbqAeTWm5fV", "proof_sha256": "6d80304c05ddb44635420604a275e10baaf3b79e8adafc06d7cd4611e114cb5e" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "the trial court correctly excluded the character evidence under fed.r.evid. 404(a), which generally prohibits the use of character evidence in a civil case to prove that a person acted in conformity with that character on a particular occasion. while there are limited exceptions to this rule, none apply here—the owners good character is not an essential element of his insurance claim, nor does any other exception justify its admission. therefore, the appellate court will likely affirm the trial courts", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792955103-6dBPJAGHFXrB6fId2cRz", "proof_sha256": "8ffca05b9a0d5e009b37ea57dd494ed439e795bc5bf90e7793ec9977881b9ff5" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792955120-HoNAHy7kn0t4LhoIQpaP", "proof_sha256": "a19695df32bceeb4e7863c3feb812032fe680cc85fe5b1fb59ad5cc35117a39d" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792955138-dURnuPupfNHFkSR049jf", "proof_sha256": "04e2f10ccf95abff581479f791e3798b5cd23a40b8f2e7e402049a670c580b1c" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792955156-oBWmUiRO6msR6wg9nsyT", "proof_sha256": "b19a18f111edf3c13e27ddac8ef35c11b8d93e1d1075b8c7d072df2e8cab6973" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792955174-ZXDjt05VRCDzfxzESWut", "proof_sha256": "09f01f213ec928204a1d575680d80c94eff920005e05bda38139f5a95208f05c" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792955190-KA8OWtRMYtD7bI7EKZ68", "proof_sha256": "580a0fd6b9847f68facaa7b49435c5fdd0e9624009c36cf70e468df9bbe6c571" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792955199-uv4jHU2nuaiOeW6HtTTD", "proof_sha256": "f9d997ea9c15946c5f17fe3bd067bb831f0a757c6e7c6cf757300b8f0e5e4fc2" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792955218-sfHgxWdUjzt5yCIduKcl", "proof_sha256": "7c1034d82dc5ef62e63e9d11657da81d28d44ce486db675b74676d097bad33ff" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792955226-kqREArRHKCU4ID2oRAdL", "proof_sha256": "4c8d68d45c50c4858b5ff952e5ab3177675f2331de39208f7de55b94a6d47611" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792955236-mVnl0v9F7xLOSiq5v4nF", "proof_sha256": "1b8d866f84cea4728d995a5d01b41f94594d9ca15fd0b905547e66e71d62a751" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792955244-DOoUSkiHMS1WD5OCwlVN", "proof_sha256": "94b90f6efd7fff40c54de1dd167735da1b9d6c548933edea828dd9def0dd3b27" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792955254-4K9tniwzeXKHv76EheXp", "proof_sha256": "bb883eed6084665ba9c9bb0939e6e7cf53d41685fc7598a399145cd6a444288c" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792955263-CCqTK7tt1Lk99F88KLjS", "proof_sha256": "98be81263689ebf56c6f44f24a8c545c3ca3024d7732336b886a3b8154dd291e" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792955275-PmiMaRwATLNvBnCvxjRr", "proof_sha256": "501b0afe7eb8df9e84849413c2240ff42b8d2f82c4bcf756b6fdf0167e48fc67" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792955283-PKfLdezgNYaUITwVVP2y", "proof_sha256": "b000a758f0bbdbad7123439754349dbd5a5f6e9a2f63e1542ad8474c306d9ca4" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792955292-mYa0XNrvuTNMdllzPCiu", "proof_sha256": "b2a62464c6a22920d46b9c491c178ddf005b740f55e9e4ffdaec7f44cdea1b49" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792955303-7QVGEyiF4yO9bS1VvE8g", "proof_sha256": "363b1479bc85f00330ff7106f6698b5143bfe56844e2435ed13361cde26acf62" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792955311-KQCHjg0I3QU9OHOqHTzz", "proof_sha256": "f2365bc2e3a2ae95348f0e9df629cfe5428f253255f8ddaf8cf3611af425e1cb" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792955320-pxM0z0L8lWljTN5gsVOx", "proof_sha256": "64233e429f15b8814f9c446f6e05a49ea5b55ffbaf1d2a8b9e7dc2f69638e977" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792955338-oHz5geR0nNtX10CDiWtX", "proof_sha256": "16d74e02de7c78d62ed2867dd13c288d2990b0b87386316c4546c0555f0b2669" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792955387-QAHDxk7LYusoNdluVw5a", "proof_sha256": "e059d4efa0addb16f62a3001f228e46598e82acba1ce22cc4c82bfc2cf2d7696" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792955426-OCWjVAp74UYUWYiZASwf", "proof_sha256": "2c9b1b491277720a21cbcc997c9844d02093d64c7fd515110a8f30713eb602e8" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792955477-nR4RWAxAhBeIIUS6NyO0", "proof_sha256": "225eaf16797c740818c27170d7b67e298a952ae1ee7bfb37774c1481854672a0" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792955521-uTrC2lb7jUhpQcpL5C5N", "proof_sha256": "cafcb779e10d05fb0ddfd1b9672eb8aee36789e7033d8297f0c4b08d5da9d610" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792955540-mXHAFCCXpI1gf3wf0BGg", "proof_sha256": "43108e115e6bd3e8be832371e23a8f970adf5240bf5c084c55d6a69cf9d6e52d" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792955557-cMK843iKFCRTHq80ERC3", "proof_sha256": "82748228ccc398fd092f6f9fcba5475a7d45030c57656afafd67bf56ed28825b" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792955596-aK5hvTNYIgk1GLvIZbS8", "proof_sha256": "97918a1d82b48d8bec0d94f6efb8a3566dca2c345663dbae48fabe1a36d128a0" }, { "model": "arcee-ai/trinity-large-thinking", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Arcee AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792955636-o4CZZcAQH8QF5SfJIkOF", "proof_sha256": "58d06d934aa0d7323dc4dd953654a2762633cdfebabb5931fb2d4f650eadc2b1" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "the sum of squares is $(au+bv)^2 + (-av+bu)^2 = a^2u^2 + b^2v^2 + 2abuv + a^2v^2", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954256-VfeaV6NN8n9UjhbhUVzs", "proof_sha256": "20bb45e8f0be75ce9f853a87abf005b48e16a19504899b5f627bddac63571d54" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261025T200000Z", "generation_id": "gen-1792954291-1LyvWgIE3oY7sQWoI4Oi", "proof_sha256": "41f5ad94a4a8dc26880090762f5282aa021e36e04011f55253d0a74fbbc46b12" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261025T200000Z", "generation_id": "gen-1792954323-MS6D8ZLFTCXKDQIzVwP0", "proof_sha256": "701f50c6d3eb578f3da9a0697b53ba4e98a0fd292364db1312b12e1ef913c68f" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261025T200000Z", "generation_id": "gen-1792954347-NX7wJWXln64Ge6lDkVX3", "proof_sha256": "bae1d3faa841ab7cb26782f5bfd7ddb969a70575bd7a600b6a3f1f8ab349acbd" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954380-NJVjYQXUSHCb5unqZSeb", "proof_sha256": "880bcaa760582a197e946da0d559bdf9fc738fa973c31816cefcbdde1c38adf1" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261025T200000Z", "generation_id": "gen-1792954412-VaNLBgn6lwFJEUrNSpR2", "proof_sha256": "df0e094bb06efeb5b2dd7f181c933b04dcb5e4af9674c8e1fc05595d7a451b98" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792954458-KtnCB2Qil0antglUuQO8", "proof_sha256": "c9b9551854982383454a41ebddfc316dc8ef073c31f55c121cde138f57e65072" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954487-dgrFH50FeQngwf728C4F", "proof_sha256": "c95b709a57414bcf220b1a2d7468d39666ca9a7e899e82a72caf09afbdd7139f" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792954548-kUlinNIe1tfoygvrreA3", "proof_sha256": "3eac04be11beba9e0a236651b64f6f7958760b5da05b4f13814006a1316a9e5f" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792954594-zO1QvM71m1LVrALqgZPM", "proof_sha256": "78dab39d5bbccaf2cb8f2bbe548cc34e5c6410688af157c5bf4c451a473be38e" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792954638-okLNsqs2UqQeJwCFggWe", "proof_sha256": "337bcd4936153b0cc4eecbbc1b2d585b969c8497daf817bbb13a3cdb5f0ee772" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792954666-wpgfUWBMf4tkERHFxIFh", "proof_sha256": "fe50cbb0f336422754406badebb47095512216d2ead4f22c9831c0fa79d5beb0" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261025T200000Z", "generation_id": "gen-1792954703-R5wbO8DWIDLCmVOUxwpp", "proof_sha256": "070fd365f68ebe6826a24b6f2b6791b29d3079b48778e3f5a27b2c2c96e80b2a" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261025T200000Z", "generation_id": "gen-1792954735-EREoA39JFd2YUbic7v1e", "proof_sha256": "674115c834150b529d4b53010b0baa688e9b1a971ed4256a97bbca8aa16a4609" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954761-3ianBPin0V5esHgxzOz1", "proof_sha256": "a636382d6464ff9947177a417fd8a18e76c52e90a91879c89eb6ee6424e33995" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792954784-tIUPY5v46hqOiMi3zi7k", "proof_sha256": "7fc4f72bf7960cd64f30d88bc715eacc3ae2ce3b2a7bb5f731697cc1a37300a4" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792954843-0y6xXR9a3KeJ6dCLvIda", "proof_sha256": "e541da5206f3880a74044fa3f8bc32f053b4b92165a500755a1efd56dcf32b3b" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261025T200000Z", "generation_id": "gen-1792954878-D8FmyjT0NJ0HnPItmhZf", "proof_sha256": "16913664b7e58fa3786f077d805f32771bd729e9fbbc8fab3479485da4646f34" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954912-kMzMPdzP3lBQwFRzSE6Q", "proof_sha256": "9b6c311a917ca51efd613c13ad3e25c60e89ad13e93bcb1e0863c30869d9516a" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792954935-LYowMnjQHDxNQvpkoy3E", "proof_sha256": "54e4d840562843c52b029238e0c3ed64d74c9bd273fa765370f404f2a0b17f33" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792954963-nARI7qZzjiGbHiOQ32Tv", "proof_sha256": "0599a004f12ac484dcf848eeaec8fba2895d052a27bce91029c374db447676af" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DekaLLM", "run_id": "20261025T200000Z", "generation_id": "gen-1792954991-cCCEHemNuCvAX32AHFrk", "proof_sha256": "392df77531c780a3fdd958270b6c71ebeb5ab47f8af5863e6a7c2ffc356bf7b9" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DekaLLM", "run_id": "20261025T200000Z", "generation_id": "gen-1792955010-W8ZdbzNR1PJRotPUrvec", "proof_sha256": "fd5a1892d6219411c3730729de46bd3ed685fd1584ab55d4332ae228c4881c1a" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792955027-kJK0c3WZFAr4ti2qfqiA", "proof_sha256": "70abcad1e81cd8fa9af319b6a9da537123d50238e124d0b5a40d501ad9f7c89a" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792955067-bgstnbwPii52ROk7ib80", "proof_sha256": "df4a7bc92aa8b9baab0a3b146a1e5d84a8708269206472a6b3941bf034420240" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955094-j5X9S8fo7eRW5vfvzChf", "proof_sha256": "20866a09bae482f5b634a3024dc19afbcfdcf31ca13371e5ad757d18d884450a" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792955116-7CUao5LmlR2TQGCtwcJa", "proof_sha256": "d5b5d115ee5e7851034e04c45a101547dad19cedc27f68e9e5e608bf4d492510" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792955153-FNKnvDFRdnXkp45nuqNt", "proof_sha256": "b6ee610ef83fa494b356010941dc517632a68853ed3c5de1235e5e5ee742c4d8" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955195-noytZmZkPBnZ7XOOqmQa", "proof_sha256": "70d1856bb1451558b0c0f2b4f72c3c729ee6d9478a30e0346e78313e38599136" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DekaLLM", "run_id": "20261025T200000Z", "generation_id": "gen-1792955219-b0WaZtK2nKkFtzKCthtp", "proof_sha256": "e09fc28582d83d38fa39ac39426f132c0daf9ba7344153f24f8ee9c0f49cfb41" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792955237-OotCWyTeB7jjXHKQ3ugW", "proof_sha256": "7c4bbd2b01572d508c0b40b62994d7ab2334326acdbdc6976133aaac54666e8d" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955294-NnJ5p7SstdtsuTiOJDpN", "proof_sha256": "90750d42c8dea054914d9011a3edaf5a400df5cdfb9337840cb74ca84f467af5" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261025T200000Z", "generation_id": "gen-1792955316-Gp4vCftR52Dh9nvMEnZW", "proof_sha256": "8db679d8edc92155a93fc62f75e683738e298c65e44169117102a53177533ce3" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Phala", "run_id": "20261025T200000Z", "generation_id": "gen-1792955373-7sPDwDotVX0yfDwSUKc4", "proof_sha256": "bea4c5ed6b31d51564760f4967d486b763acc4bff6af69516a16f1427030d115" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DekaLLM", "run_id": "20261025T200000Z", "generation_id": "gen-1792955435-J5JjCL69NEpHnyfNSFE5", "proof_sha256": "20b8a7000dd055acc8bfea43e6f85202ccd0ba7a5c04686920c65d788dc33a0c" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792955486-49ljcmu6CBD6txsfdU48", "proof_sha256": "963371751189ce49d82afb32e42527bb0967432cc5717240f4726cb60173ef71" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792955545-S8aej5Lqj1qjzIr8Mujn", "proof_sha256": "06e694131ccf0970507cb4d2a675c69039b1dcec51e6b9d4c2719bcd87a2987d" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792955610-J6w6Dnkfnq5bOOZzyWlh", "proof_sha256": "c25ae87533579e139add2b4cb6ac05796472135083624326f377cdbbe9868947" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792955669-JYDPQcrII20wBO8aaKlM", "proof_sha256": "938156cae0c32179f28af8cd7c05d010e3ffe618a3e674654bcdd69e207835bc" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792955797-bFvvSQP448LLjYUAjSVU", "proof_sha256": "ad9ebec16a0ec9c4073be12a8e91d38f8b5c115d6026687f1e2e81f376961edd" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955858-1Vh2OAz1pwkgFtLONz3m", "proof_sha256": "3e723f37dd3e8fa8ee8e48f0943341e5ea062ceb56a093b1115bbe98359c95e7" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792955913-FpsUdEijSyq0nOkoqSGc", "proof_sha256": "aa6034ca227b654a33b21791e5790b3ddbdf21607588561a6d6d3da764ae6437" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792955986-GyVGAxRJSCPhkGc1D4cI", "proof_sha256": "6c747aaa148e54862019ca7a61deed19e2608fb764a44429c4ca5758e0e61134" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "moles = $153.75 / 37,000 \\approx 0.00415 \\mu\\mathrm{mol} = 4.15 \\times 10^{-9} \\text{ mol}$", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792956015-LIonw5dtPzMCMNJZ8AOc", "proof_sha256": "b9670927b646356576e0579b2d9b1e44782b65443446d453311b9dab47bd545d" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792956044-UhJJUwQh4AzFhUtWIIew", "proof_sha256": "ec7f587b4e6d86ea822e9d253e387714a342347369eb7a9ac5ad57e2dd02f657" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792956077-LwWOq5Y4ohPouOZw9REl", "proof_sha256": "06d4c11f07696e8bcf77300499dc6e65a8f394ecadcde578e74051dc68b45a64" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956103-P0A3odZtcSAXwdPZKDgW", "proof_sha256": "b5e8317ebdbc1ca33b7382fbdc0e20dacfda7157e7835ea6e73d7019ca6cce64" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "numerator: $0.30655684 - 0.16973319 = 0.13682", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792956135-8laqKFfaY07tRpzXsl7E", "proof_sha256": "64d8fafbe4036ac627073a199f63ff10f38d84a8080e323fcca011cfb8867d20" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792956162-JpRAyEscboa2hCMLOjdS", "proof_sha256": "ac893344a3c90295ff70b779d83b8d36f3426e3effc291909e86378136369083" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792956197-N3hE0m5UTRRzVyxlPjpZ", "proof_sha256": "81ccdf7ec89564f4d2aa04a33d1b9105feba179761e6b4a9f18eeeadc448955e" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956218-BoCaOFB2fomtdSBV3y4z", "proof_sha256": "d2366acd72a54cce4bd3e66a69c6980a95ca3b4033ab1d5f799a7043e04f7d68" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792956262-gIo8wIdkE3rKuU8rORff", "proof_sha256": "27a649702c04ef630532b9b49763ff8edb2b8923b955c7331b231dd1c14df7c7" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792956292-UlWUT9G1YtvmEcYnSwc7", "proof_sha256": "4965d0a7b0cabdc6990cedc600bf558a4365cf8dd32f8a8219ba2adf1d7fc6f4" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792956315-1rLNVcjpzx7rR78hFJZd", "proof_sha256": "d587f8c89cf47ac4d8fe43d7b367c6a14062f3ae4003785f143c928894648e9f" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792956336-spV15A2I0dRgW6yZzUo2", "proof_sha256": "bf6ee68978c7c0e54e6438724db650685c7bfff42fda177b03cc82025013b60e" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792956355-pdtvvSY291E1Bv1AkZsR", "proof_sha256": "f85a40ca54037fa7b40d1f1706329b072adfd9e88f125900c53d4ef3e41e89e1" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956375-KVNnzdIwbICt9UDloYnf", "proof_sha256": "20db5a574e4deaab7d589e92ab96f00492c1401a7df4a1a00648a93b3c5701a4" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956407-dGZTrKkgzNZ4jRATsjaa", "proof_sha256": "50507c464d530c0f34adb71674232fd7fd71c513f39e93b3bad175b0f87a9a90" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792956442-1PB9OGfX61aZYcN52oyO", "proof_sha256": "74f613bdb1f7e8f8c9f743cb3efc99627a77723a2d5f086f1d0de585d0f33c0e" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956472-ghGWyyDSr8gZFLvh4dkB", "proof_sha256": "d069948c40d6b83054d1d0db08211acc146d0c8611491b5cdfd27c18f1d2dc0f" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792956518-ct1VYChU0wTagb7sI2c4", "proof_sha256": "e09a0a3ee866579ed481840af079b9c95ac6239dfaddd0cf69dbe203f05cb924" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956542-bsCQWM9iNY0hZrOLvEZV", "proof_sha256": "6a78ca19f17c8da28e2ef921858134c906f5a18fbcaeab9a2659be5c8a0b0618" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792956564-pYXcco8hDX1gxtJbQ3Kx", "proof_sha256": "fbadf7c6b2ba19c84dfb70705345432c856457af4078fa4f48680df9d49131bb" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956596-JkJrgQqHQ7s5usKAUkxv", "proof_sha256": "9ea45ea2ff0618872acb87600be8c76c27507bab41ea11c526da86a324ca6a6a" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792956643-VIRyjyXWirmc5Kf6sILF", "proof_sha256": "740ae78004bbed5e373fb5173818bf2dade7c59da8d010a4c49c65cfcc0fd26d" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956711-Ia51vJ5ja4vLqRqfWMAg", "proof_sha256": "f430e783b105fa45ddf931aa3684b2f59ab16b38d88fe3c75052dd75f815126d" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792956735-JJf4GnsxEIxUBOwD0Qxn", "proof_sha256": "6306e98f1c77343ddf9e65722229c0f3faa99c44411ad26a207ec1ef74d2f08f" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792956756-547QnpUCraBvD8QjtuPs", "proof_sha256": "4e2ac6cfa0e149120537de08e43d61937936b343439ec52c2a8ef83a0206f62c" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Phala", "run_id": "20261025T200000Z", "generation_id": "gen-1792956779-DLMXmyVq8tgY5RFB5jpN", "proof_sha256": "f6c0359a8f1306905a14008cbdccc9df9bc4018b6be278dbb3f70ed2d79b9fd1" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792956800-kPQLgCW04aMOwEPkSBm0", "proof_sha256": "6d6a44c439453af03c4f6bf72803ec88fbc299cdc159948c71b96e220f6a8b08" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956820-QAUmXZrUpnAk8q3ykUC0", "proof_sha256": "2b49a6afa4277058229de0ef2b1e22470a51e0aca7523563c534085258ab57cf" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792956843-Zv8dXt8usmIya9F4DKvS", "proof_sha256": "aea682032b3d9a341b4aee71d55206dc0afd33f8280c788bb5befdcc1b303720" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792956870-UUU1HxMRCOtnGmzhFmQ6", "proof_sha256": "8f3bcfaf5c431aded5539b80024d804c17f59db257783ced74947b1885070231" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792956898-HWsRUWKXd4O229zGgzJh", "proof_sha256": "3d269550c27db937e814939f3e279b8365e4727d4d0fafe0aaa5deafc98f8e0e" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956925-2l6zGKI9WaMck5LkVTev", "proof_sha256": "93f8d01ba4386ac9b0e461c79b492dc43cb53e056c62f4e7f6186f2e865203c2" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792956976-CPUvBtPtB6Ql5aG9HnZR", "proof_sha256": "c35733579b77c675b98d22310caef3d2565d12daf8b2f002a70ea35194dce496" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792957010-nBHiPp5HNElu6bjmtsuc", "proof_sha256": "538f3f75e30ca83df510172797002aa4c498fc61c547bd2a767949fb937463db" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792957039-rmQVJhtuTqATNQQMbNlZ", "proof_sha256": "cd5d0804a48f1702f9f9215d837cf0c28495f5d0aa1e10b48593dcce6c0a5a0c" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792957080-DxFBWJD7XJSGASHVSUF4", "proof_sha256": "165d538293a67cb236a1fafb7acc505a780c5253d3ce272ce8ff565c8af74aad" }, { "model": "qwen/qwen3.6-35b-a3b", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792957097-AtVd2FaldBVGvhJhRbML", "proof_sha256": "bc5785e6f7a21f5e8939c52c10bee5b7d78cbfd7335161be1f754ac2bd9aaa2b" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261025T200000Z", "generation_id": "gen-1792954256-BiOvBwNZ6a1n3s0Y9Rfo", "proof_sha256": "1cced73bbd16fc1ae65c0022c8912c98caa7016f02c182c891d37dc23ebe0c58" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Minimax", "run_id": "20261025T200000Z", "generation_id": "gen-1792954273-Q3U4DbeO1HkfR5ePmCJz", "proof_sha256": "e77e8ecd33dfbaa729df37ab32462daff4e612bb3d798a4ff6f16e03a53cf933" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "ModelRun", "run_id": "20261025T200000Z", "generation_id": "gen-1792954307-eLcEu6fZb5jwpppihLtm", "proof_sha256": "9ee838b8efc48d3fd1414b1867ba5168773c2b615934ff73c4d933d8a33ee91b" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Minimax", "run_id": "20261025T200000Z", "generation_id": "gen-1792954323-QLxA6YFlP0feciXvPvfI", "proof_sha256": "b7a7727a8e82bbfe656e0c3115cd78dfcbd9f2d65a6f085622d666bc228077a4" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Minimax", "run_id": "20261025T200000Z", "generation_id": "gen-1792954346-4TEVXmRhllBFquTltXPL", "proof_sha256": "44b3430d04a3401c6e106a7927b2f2872ef47c8d93789226d9c168c376c92f68" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "ModelRun", "run_id": "20261025T200000Z", "generation_id": "gen-1792954382-Bbz1tENm7dSlmplR1uTV", "proof_sha256": "9a0772e48fd95b162b0d1cf20ff4d0fe9cb691e3893d88d71831124b9b0dcd21" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Minimax", "run_id": "20261025T200000Z", "generation_id": "gen-1792954410-BgG951DFTWR7msNFFrVT", "proof_sha256": "547c31cb39756112ed06fed86230ed71098bd265b767a78992ee7c5827cde26f" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792954423-HNwMIBKbueLCE7u2za6J", "proof_sha256": "0513e2a78d783c2714ee391b4d2245dcfa77d3bf60e4f76530d7cf41c18555d9" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "ModelRun", "run_id": "20261025T200000Z", "generation_id": "gen-1792954439-cJ1IPnpT0SNy4B1f5kLh", "proof_sha256": "a86bac443d798bb49cf9c46bad9e521e1b44bc53d3ca2f0d180f459f7940a175" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "ModelRun", "run_id": "20261025T200000Z", "generation_id": "gen-1792954460-qqGlFnaVMDdQ9SqwaiHo", "proof_sha256": "eac4913ce5255a8c4cead05cd8cce337773f492b003ad51678d9f4053be8a283" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "ModelRun", "run_id": "20261025T200000Z", "generation_id": "gen-1792954485-G4sgFeiOLhBfGNw47PGi", "proof_sha256": "85a8fdf0636a8eefbe117dc739bed9c803bcc899c2609ac5ab3176f6990a4f12" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Minimax", "run_id": "20261025T200000Z", "generation_id": "gen-1792954503-elRC14HgDNTwEGN6gv89", "proof_sha256": "5ae3f00c39881821c0089af9dc61cc7d77f1996aab18599fc567b1712f6571fd" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "ModelRun", "run_id": "20261025T200000Z", "generation_id": "gen-1792954546-MtQnYlpROrh91jqtVChf", "proof_sha256": "07f51d777dcfb8c7ce29b2b907cfbf35ff7086bf9f7952c9e1e0a0778a98ac68" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261025T200000Z", "generation_id": "gen-1792954564-6vLn8SXsUyolUR75259m", "proof_sha256": "c294f405d024a710784a62e98e78ba6b1f21122580b6f3c8f3b9ebac50ca828f" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Minimax", "run_id": "20261025T200000Z", "generation_id": "gen-1792954582-fMjG2rtnDEezOFZZ9lXc", "proof_sha256": "8dbdb4d6361984537fc9cf2e5d752da5aa0817fb53e30a588dc5941ae05c0457" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Minimax", "run_id": "20261025T200000Z", "generation_id": "gen-1792954596-0Efn2BEAdh9E5y3YAeo4", "proof_sha256": "e8ad2b99f57ba0c2b40f38bd7f5f6044ed4ae285e29dc7fc9f357a4ee74fa9f1" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792954626-GwRHDETbvtkhNxEUWGwD", "proof_sha256": "152013fe7389c3a241403037c35dc5e16917b15e51a74abe3b1bba442154a316" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Minimax", "run_id": "20261025T200000Z", "generation_id": "gen-1792954636-lZNXWWGXeymlQNizVbfb", "proof_sha256": "8224924b39ab7858c3ff36afefd40324d2e2a63a50e1f689a1c8502c57dd4e9a" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261025T200000Z", "generation_id": "gen-1792954653-Pb15gae00TZ1y7AUkF3g", "proof_sha256": "3714e29ce70bb72d54b9571fecc8c3bc3ebc9316ca4cf7280fbc8431a9895b85" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792954664-hVFGY1mSM6cn4HN0wh89", "proof_sha256": "bf3ff44cf3613983f467b9de6e3fd0325cf1453892b81513d8f01bd848d71e56" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792954673-O8ogAoi5tXPzqh7Zv1Js", "proof_sha256": "a25c605254b7bd7b4e6eaf6098db9e85421973cd338f00e59cc6df97975cb993" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792954682-cagoZjBqSYZtwymyXbSf", "proof_sha256": "ee4b79ffaaff3a8fc4c512faa97c3502903c49eea0b38650b37277bb12d11f02" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792954703-AbEJswbBIEdOE7JhOhBz", "proof_sha256": "dd8d8d460775a25cb5d50c548e5e8c9d47a5f609f8a73ed4ce220f8d8a39d9c1" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Together", "run_id": "20261025T200000Z", "generation_id": "gen-1792954719-D0Bd7YzLj8vlZaOXo2uX", "proof_sha256": "b83478534d6f88013c1e07809bdc5d873621bd3597290455b91e946c3ca9c9be" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Minimax", "run_id": "20261025T200000Z", "generation_id": "gen-1792954747-C9rkUcMS6dHdrZONdajN", "proof_sha256": "19ee47b38985cede0a4f358f5e2672afdc9d237335290920b79e1cdf90d24973" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "ModelRun", "run_id": "20261025T200000Z", "generation_id": "gen-1792954786-VaUlMOM97J427nw7jmpH", "proof_sha256": "6ac4b6723cb7d1d5173730dec386bdf26cdbd8317a6860668ac3638452e33a37" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261025T200000Z", "generation_id": "gen-1792954810-uJ6PNEsmoeEUJWorScTD", "proof_sha256": "da3906fbe99cd3db23947bf7c4af8686e01a5b2c67f3619bf31505c0443533ea" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792954827-8tdKw3Mn5npQ20cNMVgk", "proof_sha256": "a03ef53046d9324569461cbce4a8709b2d4060fed08d172d7e15d5406fa22f35" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Novita", "run_id": "20261025T200000Z", "generation_id": "gen-1792954844-be25Q9G99FLsKfGdp5UG", "proof_sha256": "24f62240ab3f80ee5e253240bee081bfb2858a825df47cf32199218dbf72f3c9" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Minimax", "run_id": "20261025T200000Z", "generation_id": "gen-1792954886-ax5rwkg6FS3czHUk5t9X", "proof_sha256": "ff580725acb50100af2f9abcbe277846ec601cb180c175e6474380f47c479081" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Novita", "run_id": "20261025T200000Z", "generation_id": "gen-1792954915-o8Mc3TvcLQYoXKifu4xn", "proof_sha256": "c34d0fbdb5da2edc284ecd694f4ec2b0b229d454bbdc1e0cbb0b1f94ebc18a6f" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792954955-wEEHh48Nf21ASi7tE43W", "proof_sha256": "3d131050544596eb5f801af0eb4879a8b38f5a920e56808da96a20090ec77bed" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "StreamLake", "run_id": "20261025T200000Z", "generation_id": "gen-1792954970-gccbbbb6ubHVScaHcHig", "proof_sha256": "ddd8a7c92eab6c1a0a53e089f9bb321c0a8f7551fb9c8b7d048961736b1a4ff6" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Venice", "run_id": "20261025T200000Z", "generation_id": "gen-1792955017-pckcpivLalYr9Q0cYeUR", "proof_sha256": "fa6a87261eea45d627bc2dd2edd8e8dcdbfe1a0598e22125421b97d4e513febd" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Together", "run_id": "20261025T200000Z", "generation_id": "gen-1792955042-tjDUVQftrKZvNZziYBRX", "proof_sha256": "36169e689c11adffb351f3d8ab519386e89657bd46e2e45d6ec121c558c0a601" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "ModelRun", "run_id": "20261025T200000Z", "generation_id": "gen-1792955083-HRw3sgtt9ox6t2ygCm61", "proof_sha256": "0f6edfd9bfaf86d6299bc3118c14a221cb6d35cb4d29a44725d048f4ab1fdfb3" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792955106-ZWr90ZcFw72nAASD2RV5", "proof_sha256": "a94ff67fdacc3190fb196fe837a04637783a0f36b40d00489d672296f70dbeb9" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Minimax", "run_id": "20261025T200000Z", "generation_id": "gen-1792955118-CDlYjNribBEtHZ5j7a6a", "proof_sha256": "7d76c45f6a89627f5a2b4e1d7d231e16ea263d56d6c622dc8e31d9938c5f38d9" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Venice", "run_id": "20261025T200000Z", "generation_id": "gen-1792955130-Zaz8TViasTdQzlRjYqqM", "proof_sha256": "83cfcad772a20b16c564030dedca98dce666d893953393b54186de3dac869264" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792955172-z6zCQsimzL2F9ab76nH0", "proof_sha256": "6d31b8cacb00a3b8ba6e0a085d6349954d74e935614c6ea161ee1a8116ce3f54" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "ModelRun", "run_id": "20261025T200000Z", "generation_id": "gen-1792955187-7MoC0DLYwPOT9KF7Vn7N", "proof_sha256": "40ab8ddcf855c7b423e2229e411fb432d56bbc1e5c0958037e5817e43fd42768" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261025T200000Z", "generation_id": "gen-1792955208-NK5OFOkrpvkDZJgunHnZ", "proof_sha256": "b94b6b9af620ffabd10c209dd714f592acba326f99b125a56e058a72c9e9c696" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Minimax", "run_id": "20261025T200000Z", "generation_id": "gen-1792955237-9BIBJKfIQgJKiOEbWOfM", "proof_sha256": "39f12c7822844c91dd418599cd62608e27dac19e6eb56f82a70af3d58a442855" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792955276-DN2dGuOND33ntQpJbbv2", "proof_sha256": "d4af7a5d1e4a282769fff1a9b48a91b802830820ec32e60687d5292b15a80b17" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792955292-bYqOQgwqzJWn3Gz924VH", "proof_sha256": "7486fdb86a9960b3f1cdadcb1f3fc396851eb74f013a2054309926707528354f" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "ModelRun", "run_id": "20261025T200000Z", "generation_id": "gen-1792955304-HCIbtgcZWhsAuhzcyznn", "proof_sha256": "9c0d29bafaff6e2b2b6d7c5f9e7e55d76a7f53b8591badeac1be921234228a30" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261025T200000Z", "generation_id": "gen-1792955336-1HjSbxTrMtXSE4O2IWSW", "proof_sha256": "26596cb14571d7ca03e932318195291d3c8f386237dc7dbf0c70b1242187dc94" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "ModelRun", "run_id": "20261025T200000Z", "generation_id": "gen-1792955348-OfRQVvWrEOkMryGflT4e", "proof_sha256": "e985305d3d1a9f9c0aeb7657c10abd59d5741ec87d635c6ae1df9b401e2c4a37" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "StreamLake", "run_id": "20261025T200000Z", "generation_id": "gen-1792955400-qYfbLJxUHMPXXpggoFXz", "proof_sha256": "21fe7b26c1ea7a26d6083daeb4edb5750f93f9d11fc2a794d46936ac63ffc2db" }, { "model": "minimax/minimax-m3", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792955473-uprTXDIOgwaKt1mn80wg", "proof_sha256": "5b143558c5051d565f18bb6b6e1c984f9b186ea7cc530edb8e6b05945934930c" }, { "model": "minimax/minimax-m3", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792955516-VqpFGN8jivtQqwvxvqGg", "proof_sha256": "9ea84bbb777d300985510beeea6dc03750e31612c9913325d4449cb206507c71" }, { "model": "minimax/minimax-m3", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261025T200000Z", "generation_id": "gen-1792955540-J6vHcov1eEIvlmmpl50x", "proof_sha256": "61d860938a881f84da565e6628935820313ac72091e3bb1e34b12abcf3ee34d0" }, { "model": "minimax/minimax-m3", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792955557-RsjGl8Qj2DWahHlyO2pe", "proof_sha256": "ecdbca8495211815550720590642ad0a612a3bee210d812d43d4e7837e4ae9f8" }, { "model": "minimax/minimax-m3", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792955599-lbcmDMHPPjQkqunR0yQZ", "proof_sha256": "0fb44088e7b05c7c749b3d093a43c74ffeaf3b0b1dc944878feaf6d4340a517e" }, { "model": "minimax/minimax-m3", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Novita", "run_id": "20261025T200000Z", "generation_id": "gen-1792955633-VggBJYcH6x5pW8Zfvu9b", "proof_sha256": "a353632814b75236c1ed39b6542aec09ef1e83c09abaa44cc24c82ee0c7ac21b" }, { "model": "minimax/minimax-m3", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Novita", "run_id": "20261025T200000Z", "generation_id": "gen-1792955714-KFufxIyPtwjPCyfJThkl", "proof_sha256": "26c8e35368184f515a3b70a5904b5268aeb65b466e592260a6b9cf68f1a466d2" }, { "model": "minimax/minimax-m3", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792955797-3qbGlriRB7cTUtXBA5v0", "proof_sha256": "6dfd634267eeec06a3d901fd33f0f446ea5ed9eca6488ac44f8692cbf09fe9cb" }, { "model": "minimax/minimax-m3", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792955802-mwnyEflT6XrJ0FVv9IqR", "proof_sha256": "49a921e0f8d568e02063bf99149fbc8a7f95a094a5eb4b30db8e79b9aae4da28" }, { "model": "minimax/minimax-m3", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261025T200000Z", "generation_id": "gen-1792955842-L0dvMRQOGhClYOu8rTRb", "proof_sha256": "1319ae55b5e3568e937698ceb1bff7a3249487df8eb4c91f352877f91036838d" }, { "model": "minimax/minimax-m3", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Minimax", "run_id": "20261025T200000Z", "generation_id": "gen-1792955886-EdDQt6rPMwwwW8ULB2Ub", "proof_sha256": "cf4f0b71dc84d58ca9a63cacc88aa9cad09f72125f833e2dc817a078a27cae3b" }, { "model": "minimax/minimax-m3", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792955931-CB4JN8INdlwOVVn4cZGN", "proof_sha256": "5e01dad76b4fb01260bdff5f4f5a8daa774faa6063891098436b36987992dcd1" }, { "model": "minimax/minimax-m3", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955973-EsBGFF3JiZ0Pch0zFaa4", "proof_sha256": "e38ebb05a3ff683ff0e48f0a46707032a4c0b7879167180b9a8e950dbdcc39c9" }, { "model": "minimax/minimax-m3", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261025T200000Z", "generation_id": "gen-1792955999-uVX0m68EyoX1rjW1u0Zm", "proof_sha256": "33c35c0a3bbfdaa4fe90e5f5bb52aa2bae0b55058a27fd1974c8160e9de08ca4" }, { "model": "minimax/minimax-m3", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261025T200000Z", "generation_id": "gen-1792956010-HJjzh8IkP2BoweOQGwRB", "proof_sha256": "b1fc1c7f4dc1923dc1ec6ad203fcb20ad93a44dc7dd3575c77e4957a25aa11f2" }, { "model": "minimax/minimax-m3", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Minimax", "run_id": "20261025T200000Z", "generation_id": "gen-1792956021-KxSbVZkkCWnTwNW8WzGp", "proof_sha256": "c1799a393d2edad7f0206c71098a78ceec0b44e0f5f8d7faae712bc895dbf7f5" }, { "model": "minimax/minimax-m3", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261025T200000Z", "generation_id": "gen-1792956032-aOsxWebBDRhOGt6EEZTW", "proof_sha256": "02acd11e7803ce4957e347ff4ff7b9b83e734365fcbbdb09dd81aa0e44bb8a8d" }, { "model": "minimax/minimax-m3", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792956040-YvDKUZF1A9hiMZidBZCP", "proof_sha256": "13aa638f6a35e449bf65bddf6dacd2d2f47963da03ebea8f60c2219a6223a114" }, { "model": "minimax/minimax-m3", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792956050-Zm5OR9CPH2tq95DFWwUS", "proof_sha256": "0a9274fe2d70f0966c1026f06b8b474ab8333c5828675a40c9aa390c3a7d0233" }, { "model": "minimax/minimax-m3", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Minimax", "run_id": "20261025T200000Z", "generation_id": "gen-1792956088-eEU9VYCTvZRulORLZZoD", "proof_sha256": "53771b0ffbb9f58b5a2629584e414a8fbcd7bd60d3b15b98daa4ebddd6ab3190" }, { "model": "minimax/minimax-m3", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956103-xCqqrXgrR79P1U4ZzRoG", "proof_sha256": "eefbae6d197574bb165fcc49af8f4d7bca82eea1e6f0bcb2ca583ee61c7df52c" }, { "model": "minimax/minimax-m3", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261025T200000Z", "generation_id": "gen-1792956112-CibhoGIjFkvkWszY2BWi", "proof_sha256": "7990b6cfe3ee042024b86171c92009a977440c516bd0ca40be2f7efb1e315798" }, { "model": "minimax/minimax-m3", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261025T200000Z", "generation_id": "gen-1792956121-hZFOR67YTYVzwFgazHoD", "proof_sha256": "319cc7742ad37355c0cdd5c3cacbe32eeaaad2eb17af00b20ba91a3a1aed3bd3" }, { "model": "minimax/minimax-m3", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792956137-bj5w736bDvmcH6kI4gbU", "proof_sha256": "8c9b59b6eb359dcea23d64300899d6221eec585535a1eaf7deb24eb45892b797" }, { "model": "minimax/minimax-m3", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Minimax", "run_id": "20261025T200000Z", "generation_id": "gen-1792956146-zmDdDUUnaW8uqxfJa0iF", "proof_sha256": "daa4b41e564a39856d5652c88c32a5a1c04e3c7ce16219961be9c9bd5d82dabb" }, { "model": "minimax/minimax-m3", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792956168-6AdqrsscoBWdbwwmz7f4", "proof_sha256": "36f6b6c9a918361a43b58eefda2ebe52bc25b77177cfdc9f26750691e016f242" }, { "model": "minimax/minimax-m3", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261025T200000Z", "generation_id": "gen-1792956178-kzymDqzHwZVhvBEWnAU7", "proof_sha256": "b3c9502bce5ae45129f6e7dcc4bc0d4729a86285a4d08b4ce6fc8b1c0e983bac" }, { "model": "minimax/minimax-m3", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792956190-NEm9OvmWhAFTQp6azmBu", "proof_sha256": "9f014ddce0f2aca36c16cb1b9d0cdec1d9f29f27eb6a0f1cbddfb9cdc21da586" }, { "model": "minimax/minimax-m3", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T200000Z", "generation_id": "gen-1792956201-VigQKhZlLG5rqMfw0Gs3", "proof_sha256": "28dce44574805d9e674b7427ee5bcf420f638073c22c7f05ed4a4d74618379ce" }, { "model": "minimax/minimax-m3", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Minimax", "run_id": "20261025T200000Z", "generation_id": "gen-1792956211-CJOmCwpNtXUpgVebb5Hg", "proof_sha256": "42c178e6de7029beafd02527f56da7cc04130a8343267f9a378b312597fb6c25" }, { "model": "minimax/minimax-m3", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Minimax", "run_id": "20261025T200000Z", "generation_id": "gen-1792956219-YqGbD4XKUbvsz7fw1FQk", "proof_sha256": "a80955150e3174f127c1267d3f9d7075bccd6458b67974e0848168f661c734d3" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792954256-splJMLtzu3bv33boTKvy", "proof_sha256": "129dfb62db586cd87f45096a923492521eb6eff6273927887816f8deedd39cd3" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792954300-L2EJxCAjsUshLsI8L6lv", "proof_sha256": "f8c392104986648ff8745f5ccdd62fa790e9cc44279091bff96a5c0e213d7cb2" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792954351-Xeydy2m92sQtGRv539FE", "proof_sha256": "fb497f0eb27716db5e1c085bc44172a13e2c73935b6dccff1ff6adb9aea95970" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792954373-6QxF2AOhSYZG2d4LVpNn", "proof_sha256": "b5eaab5340276e2975077add3857aff51774d3b0659286f5a41803243475b50c" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792954424-WoTIMRRR4ktUbDpz6S9P", "proof_sha256": "09c11c641d7a3bccb5beb01ee3dde2286d06a1f45e54ea001e9a349f74e71315" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792954484-h2p8OKIMAJdwRcQjRQiz", "proof_sha256": "a8d9619a6b9ebb0dd5ec3d90ed5df000e76828e91c98f35c5eda3dcacb77f025" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792954513-nxkCvh3ctBLve6M2V8m6", "proof_sha256": "46349414b2900ad252336ef3351d7fb268195031922ddea55dccd08ed5c59ba1" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792954533-leiP7Zr2PfDLKBKyHNzT", "proof_sha256": "c95067776b6cab440c3c8a8f5a78b0dc9ea07c9e1a8537706a636140ae6019ad" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792954586-A9feFHKdh8vHdg3kMBxk", "proof_sha256": "2ca17f262ea5c8108581dbd08a8e071872285c0448532c84ef3f3020bbc0c96f" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792954617-8zw71BVPbd5nRmQ8Wpzf", "proof_sha256": "3bb2a37cb940a61931f5fe819a3bc0524b88d476fe1ac81d8e374a5c081d7c96" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792954665-pyItWI6h8s7DIluC9x85", "proof_sha256": "9d8611202303cc2a7102b84a3290e572561499a1e5bfd9886ed1d9c41a76a5f6" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792954700-Nmv3HrFCYQvqBbEV3HsH", "proof_sha256": "7d918d8a1e127604cde0d8c30ef184d887a8531d4a63ece3affa26baf02b2b11" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792954762-dMT0oiHbFSRFFU37tk56", "proof_sha256": "327ba1125cf26b408124214df5e4c2bb1acaafe1b8d012fd3370e89da658c092" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792954777-SK6BsKzOCxZY66Z9TRRy", "proof_sha256": "4db68aca4fc822f3dd80b76955d47ad2f50cb290b11f1936dbc59f3cc7ee0af5" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792954831-pxiDTzY74h1nft44NC0T", "proof_sha256": "7ba27e9d6ada64ae8c9f637995b74df2a4dfbe228e7e54201b411d25843266aa" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792954884-K2YFILKZ7tF5omHCVGZx", "proof_sha256": "a5ceaa6defb64c1255d2662dbfa6cc6bbd37b831bd4a9394f258d14e6af3e0da" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792954919-G8eyv5PwL77eVfy25OcV", "proof_sha256": "a77c8401bf43a08939edbbf56142501bc3e388dc2adc57e03cc802e2992c3f03" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792954974-W6bKVTIZ2FJ2VGo6Kmwu", "proof_sha256": "bfbd1271a0cb22d8d7fb84edb027e61d7af0f57315fb8f9265c65630e737d037" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792955014-iMk9aBqAZgUX3eImFLzv", "proof_sha256": "8bb46ab6c4c7c0ffc17f21676d445ab3c910505bc90da0693310bcb5fd5bb63e" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792955042-YNX5AONfHLzugGZ3qYYO", "proof_sha256": "cb8fbfb6355ce29454fc46f255147d122ef7843f985b49ac49892a265550a714" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792955060-EpEZS4NXEUzNFZ05cDb2", "proof_sha256": "5ebe1b4d42293dd658ef9f5d1ecb683eb62986098adb35f850cd921081170b0b" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792955076-DRjciXj5PG0zAOfg1hPS", "proof_sha256": "2291cc9f9c7be7f05e8138e9218f2b2da30f1bccd1f4febe7a89571556ffaffe" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792955118-OtZhgpSBgGl7WMyVzmzM", "proof_sha256": "910e5b6133b124bf64645ee56adf7292e9e7d6e8b5e651f0fe7f4a316cba4a41" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792955182-bIqjF64MnxsbIPSRDkTp", "proof_sha256": "4d0d2b727bd456baa730b55f3f739b557247f66dfbca63d7c04eceb51c087890" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792955235-ZnLVu1mh7IYkeBomdVF0", "proof_sha256": "aaa77d8b3222123031e84fcc6b8239b166f7420b40137da8a9e7b1d8c74e2d05" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792955288-JyHsx8435mjpahfoEtMk", "proof_sha256": "eb781e2385e2f85691144631d3bac8d27bdc6e41afa6e69728b63d8557198315" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792955350-l1ixw4VI2DpGWNgqH5uI", "proof_sha256": "27864e61b639639923749205ecea6747aa36aa8a6f3fbfcb2471fc1f2060bff5" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792955439-gSddBSVPdKeR1NczM5l4", "proof_sha256": "8574e65f299151cc56584d7179b434b5bbcd5044bfe53b7aac9ab1271e82db1d" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792955521-7xlt0sf7UmMljlUvCqdW", "proof_sha256": "3c230b99f3675e304e06ee84670f0fd302d7449f71d5d6b48ac35cacdc40801e" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792955612-OpCa7QpLORLx4hzxOpoF", "proof_sha256": "d594fb003012641f7af04aa3ad68888bee0492d6d3b4d30d2f1de4bdc57bd7be" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792955673-wHGSqetUKdO6miRNujol", "proof_sha256": "f1ac041d798b5d5da5e350a4ed39261b53688e6e86a01a319ec94c5ace99ad99" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792955797-tG4lDsQR7drPOR5eTl8Q", "proof_sha256": "52808b13819c3a06edd145711821b79bb903cbf506c91c139688d0632b2cdbe2" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792955874-jlglQCQkSchvoWHMy1sB", "proof_sha256": "17eea1f382634f177a72e5a34f094bf35e5772c9953086e65e43653697dca25d" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792955955-rB3b8lHcP0MrZa4tNss9", "proof_sha256": "7bcdfea2cb9558cf90ceba3f358d4b4a052d5c8d43c601f90eb4aac61c5890ce" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792956010-ivdmiEuhrqM4tlEEN5Y2", "proof_sha256": "04b8420e183ae68cbc9f520c71c16c6d43c84e767f8f50a279de962f53e07953" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792956066-Z6mulw7OcfiPHEjTimHo", "proof_sha256": "0450e7e2d5aa07fa6bc57956b37d59a6d8c44747bf25e3a33ae001562be62558" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792956134-0w05EeuQXygIS2aupRyo", "proof_sha256": "f04919d9ec0a06122acc79838e7861f06bf134503f2903e4ae5a3b28bf6f4c51" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792956202-OtlV5WzQg2nWwM1Pkizo", "proof_sha256": "c74156b5cd0d34d1b0d18da716e1d7d909571d84f545fc627af34334e0c65f98" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792956261-1rXTY7C7Jvd9HFR7WsNH", "proof_sha256": "ff44e7676f6c60e204b05e08f1140d6b74808eeaa7bd008f3ac0fe21160389a4" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792956298-LxGh1WsFJtpSm4d1c25G", "proof_sha256": "11efcf32ebd0e80d30637911be067e843384fe99c648b6bb97430de34447d21d" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792956356-mFvamHWJxAJOldg1CuTh", "proof_sha256": "7541f5570d8372f7e588782a001089ef871259345e0fc66b240106cb507202f6" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792956378-mjWhY7VFiPZ8gJwqqfkO", "proof_sha256": "b19ca1d8ea1cac18c7539fccee744973383d1dbc4ca91ff4b08e302e05bcecd6" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792956424-AgzcQe9QD1cPrsOn2sOH", "proof_sha256": "cc49f8df0a6e467205d12c283cc9764cd60dcc34dc0a225b1883cf7f0728dc08" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792956488-a0Z29keZTuCQpw1OvqT5", "proof_sha256": "dedd7f44b6e5018a730d01f9f52ff8b00f3774873811434df35926317beadbd9" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792956534-u9nJvWDxUGdVKh9DMxiM", "proof_sha256": "bdfe398edb359e5389d9fc6960d66d023aa9215c0900ee7819fa3c32c8ef2a27" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792956563-7YPOFA6499FFz4xdBVyR", "proof_sha256": "9ad58680173d541490a9b1cefbd9d4c0a0d554e01e942f6887d98787b8d900a8" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792956596-1ylmv8VAn2Yc18rXCpV3", "proof_sha256": "68e3b196e0d26d289422817007943f384384c2b7deb94a268f0d0021122ed91c" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792956616-D1IJvObTTgVvK14i6q4Q", "proof_sha256": "0a1163bf22c6b55ad33db257a7517995cf506d6591ae891b719ae316b5b057ae" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792956668-pvXj9cBoeFQJpmhoqC8a", "proof_sha256": "4d20e80d3473f2f6912d61e87db534df1a2f7f4e83f4c2cedc099a31fb363435" }, { "model": "meituan/longcat-2.0", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792956722-xj4eqPVqYwapLLPOjRGl", "proof_sha256": "c6a90ea8a0ff95ee8e2c4d26c3db5e2d9df43887649fb22b2faf3122929a6744" }, { "model": "meituan/longcat-2.0", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792956757-Twd7Vtbu4DwoNWzWtPmw", "proof_sha256": "d6760e47b9525973f0d08321710b0ec87fc37473138acf31c6957772cd924c53" }, { "model": "meituan/longcat-2.0", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792956806-3dmBIiJZuFrE1DzOgwRD", "proof_sha256": "4cccb8d10a06e528fc4ab601f3c5b730a067f5232d217c496ebea1d3fd29dbc4" }, { "model": "meituan/longcat-2.0", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792956825-g5Xxnmko8XLoeyrkDwAj", "proof_sha256": "79a2b6d0de0c699a3728712ad4af67997743d49fcfc68a82ab408052c7710b21" }, { "model": "meituan/longcat-2.0", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792956883-MubSkHXbkKoG8gn6FLPt", "proof_sha256": "029114f1ade8d3c110ce1079d979d79b1708466b78ec38f4c8c20891099d0c58" }, { "model": "meituan/longcat-2.0", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792956929-1ZaxTGWBtC7PbYG195r5", "proof_sha256": "8ba20b0bd0aafad988fefcaa2ce94e485a062bce26333f0c1e71ad0b840d8c81" }, { "model": "meituan/longcat-2.0", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792956965-uWWiaJK849YfMEeRsMJH", "proof_sha256": "ce65adf02ddcc4b51ee0c7d4d5dfca65908eba8e658735ab043f7a874f9c0181" }, { "model": "meituan/longcat-2.0", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792957028-eqOuoB2SgQUPBNAHW1Xn", "proof_sha256": "2df9ecf25b3ca554a71f833f96eb026eaa3fc788ac398d1e13382b5c73f25ac0" }, { "model": "meituan/longcat-2.0", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792957074-5P1DXuKe6oNtt0FfAe3o", "proof_sha256": "d1500cff8d6a13f379f8effb7dfcd08d30f32d13e8c13c7b16a47085053e02ff" }, { "model": "meituan/longcat-2.0", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792957090-ozmvJ472cU7MoBmFUVjh", "proof_sha256": "c90222e6f49d275c0b2ef3d5133e6eae9c635b74883633c0b30999d34c7cbf7d" }, { "model": "meituan/longcat-2.0", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792957157-0bbX52WeZ3cl4zkFh7yd", "proof_sha256": "bebd799e9ba1e5c20c89c776f2be731109bc6ce3ce21258083b2baf86c359003" }, { "model": "meituan/longcat-2.0", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792957178-wAg9qZidjIyzg0CTfq6f", "proof_sha256": "8e1133e5eb0a9e26ba24a4a7d12e740ec3ce977f0637535fa9b703a38db75e01" }, { "model": "meituan/longcat-2.0", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792957198-TMAgbuTFX95mjCThvdyg", "proof_sha256": "bd7f7c3dcdb9d228bba803a9d2177d47923bfc7ba1daf08897bf2904773a21f5" }, { "model": "meituan/longcat-2.0", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792957209-uQytdDqZ755gUI5gfs95", "proof_sha256": "f26e02905c3b355d44bcaf111ff1f53c02f9bf518fa9031fff8cd5ef07c4a685" }, { "model": "meituan/longcat-2.0", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792957237-5lAtajleVrTPAqYyAsGP", "proof_sha256": "76fe1dbe0bad803a81b5e42fcce3be04795a4a521a5c50b3b9a680f8cb801dce" }, { "model": "meituan/longcat-2.0", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792957259-gJlkTXr9NdIsT06nHFQy", "proof_sha256": "3bf05e1e0d57f1c6eac39c048ddbce0b1f502f8df2436c3765c19ef40caf9e85" }, { "model": "meituan/longcat-2.0", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792957271-nDiaQ09jlD00Q0QnZFvF", "proof_sha256": "088039b99607cb83ac92d9878e4d83941cb89e7ea1d7d0b7366b7d6459ad1243" }, { "model": "meituan/longcat-2.0", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792957291-w2QgG7z031K5e6ndzgVf", "proof_sha256": "eea2054a11a5583b0453ee1a755e04c325ffb4f7fd8574166f862dee87200e39" }, { "model": "meituan/longcat-2.0", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792957307-geXZDSBU8WCXLnMG5JmN", "proof_sha256": "da5bd5412143549303945d75c141b8cc7b4b6eaafe954ba452361aa03fb069cc" }, { "model": "meituan/longcat-2.0", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792957332-gjKE15vBFEx1wScQeLdz", "proof_sha256": "c385a8959aa25b5d8b327e808f779739f9eeec6f2e7ddfcf4113136372913d7d" }, { "model": "meituan/longcat-2.0", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792957356-9QrIS0SMTBoXIWew9POi", "proof_sha256": "ae97fdc5112af94227f760aa5e4d220c119bfac73003ea748866a8ccf8af63ad" }, { "model": "meituan/longcat-2.0", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792957369-A4rUZ476L6t0CyONPmAG", "proof_sha256": "239e954f4f428dfae65fa71396681876ae24ad2c935deb6f743218e7fe94a4dc" }, { "model": "meituan/longcat-2.0", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792957383-QpROZoh8NWz04TxjNta7", "proof_sha256": "4894b75ca9cfb6254d1702564a7e3a6972c0ac58cab321acaa69782e8a4b7f51" }, { "model": "meituan/longcat-2.0", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792957431-ZmQa0JVJ1CoYBUM2rwvq", "proof_sha256": "ab4986694748aa441577e81278011f03f660fb681110fad74ecabb76a41bb1b1" }, { "model": "meituan/longcat-2.0", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792957446-4ExDyNxUmoHANG2BWlcd", "proof_sha256": "551b45aaac05be9ba254357d6ab725c20cdebd74a08087c8c2cdd11851d5cd81" }, { "model": "meituan/longcat-2.0", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792957503-ENHwRwK62zGeb8kupPyf", "proof_sha256": "5be06b73972f5cf315ba758f3e098054d5329161b6335ef77e98d462164d7eb9" }, { "model": "meituan/longcat-2.0", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792957545-SjIrrgXVdnrYzoWqMBOJ", "proof_sha256": "7c7623be3845c60d26d387980cd312df7f60ef917200f8f0e033d2ba3f805c2b" }, { "model": "meituan/longcat-2.0", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792957634-Q130ttLF8o860vVqpr9M", "proof_sha256": "348a64c1d546d69db959db4d37f436436b8507ad50015bc2d4c05be604651bd5" }, { "model": "meituan/longcat-2.0", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792957683-T3jFN4mHc8oDOc7yPQ3C", "proof_sha256": "1e0e26a98a48bcaa379045ecd6710483bb1bf8a980d3e85f9dc1da9df42be549" }, { "model": "meituan/longcat-2.0", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792957698-YjmF4k4EKGBX0uYWf2QZ", "proof_sha256": "e50464aa8a3d3e6f8053b0dc160972465c9f527151ab62d98920bbe89cbd3469" }, { "model": "meituan/longcat-2.0", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261025T200000Z", "generation_id": "gen-1792957712-xrxBB1zqmZlmwqIYoCFr", "proof_sha256": "5340a5b57983ca581761f58f9b1fdba6d1f04f8a54d52df772787a1de346173e" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954256-2a1n1kxEyqjutw8CjkmY", "proof_sha256": "000e90b521dde370f7cfa86c218b811f0f6bb49ac6365fd1170330165c8a18e5" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954265-7ey6dzjLEo0P1fjr8Ia2", "proof_sha256": "8d56085cecf52e0bcac91ea9cfecc4d5749026a589b47bdfa459ae94e3c7f7d1" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954275-0SJX9G14YN082in1ckxC", "proof_sha256": "cdf3f8dfcedaaf0b2737e88498d5103c5573a4c4c5e3c85525194afee9fef9a1" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954283-diF3K3dxhKEjEG5q41MS", "proof_sha256": "a97efc4ebf8732e5aa582bbfe162efdfe06a55b8e76aacbf55a9aa24c993343a" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954292-YUlxysVZqGOL4y01kaB4", "proof_sha256": "f7ba6db28a40c71bd17c09a5dd4bbba91ede14d1b1706f8b255ef2dafcc0ec92" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954299-pdVbtAinYzMTKlBey6l6", "proof_sha256": "721c4fa6fc3a06aff23ed07bb08f113e0b47fb67593ba3749cfc2754c7ca8c15" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954309-ny4qrWGDKfpsa37pAxVJ", "proof_sha256": "ca7d5eeb9bb3aa4d792d2e4faa099f2bc978e8ac3376d443a4cddb9643c8738a" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954328-g7aORPFpnNjn42VdqtVy", "proof_sha256": "2f0aecbf220952a69d24e04924ff7f1b9c31a0ddd2ccf31f80916f7c3a94c285" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954335-dpdsLM95iVMoeTo7CcvK", "proof_sha256": "604c6d1f76598149f72611baa2f8acdbce687d7f736b85c84c8d66a07b06f18f" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954348-X0m19xlH2o2ciPuSNVQw", "proof_sha256": "5d6237cd9a0c7c2529ff5565aad10a50ad33a5ec9c237dcb46add10e45d3b4da" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954360-5omr2rfI0srZKLTes7JW", "proof_sha256": "d6ad3cd75b94c956d608cef9895d9f73e6752f7af6c344f016e0cc7d8465ec8f" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954368-om1dsZkHu83SWMFiUpDO", "proof_sha256": "acddbdc095767267f769e9837a56085c5d269d990d14c5d6a4134b183e4fdf4a" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954375-KwRhmitQoGAT1arWspgD", "proof_sha256": "2d8042452b8e493a65302729a1be55c3554bb43ed491ff0a098b00f5ce676dba" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954383-twKTLCYJS8Rp2kMK6k7j", "proof_sha256": "cf92ac76ab4e47545981d667af71a1691d3ad0050e284d603c635dc47f9d99a5" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954402-KdpLnH2k4sssCDrdD1It", "proof_sha256": "73c38b008bf767ede656080e683fc9572f91c451687d982e62b72b4d6e5647a7" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954411-JGqt0Q8ggZw94FuY002F", "proof_sha256": "c5c3dccb8bdba00f4f8d240449e4eed9d8240d411ae4e6379d5c106f6f567cd9" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954420-S8ycFL9PFFDYTadZAhbk", "proof_sha256": "ddaa178c423e95295c814503134e511281f2d0ed8deec45c30fc701ec38ea5ea" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954428-5qWmJNheZU3WcPX685pr", "proof_sha256": "415eda4d68a99c5fccd4b29f7b759f949a7c8576a5f3a6bf200be673e69f5fd6" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954437-ZJdOeQrp9Ei9b6ZJO9LE", "proof_sha256": "0732e80ad8973f35ec7f7c6241c7099d922c233c3e5e51349595fbcdfd247db6" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954446-47kKwRO2xZgWGu3qxuXA", "proof_sha256": "28c413026d1626273600723b9e938c1f7dcb32f0b062ec0451e9a61ec0f258c6" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954454-87e1U2TfWIJBO6ysSaAE", "proof_sha256": "acdb02aa2dfd2f54b023b36ff54003919b19080159a79808bc660b7a617143c7" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954464-j70pjqaYxtHn4UPhGyUK", "proof_sha256": "e07096b7d30491ed24afc4cd9099d0fed9f3bc0079a75e268714397d43aecbd2" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954477-ULItjKry2ceOX1yPMmpZ", "proof_sha256": "795cb4de2326760527405a9f26f509d8b1def1133574e09dcccf2f960573b740" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954485-OQk5sDBUoR6Msqzt7SIM", "proof_sha256": "a29f026838c2818c998b814631e696b17c0e6a66adcec50a8b2a1ea01bad7e30" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954492-8mWTSsCi6CEA8gnaa0EB", "proof_sha256": "e9a6309e0ff5b9837fa9d07b14a32e64829341139534924411face6fef55f9bd" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954502-3HXBlqUt2X5nENDe8hsl", "proof_sha256": "e1ef8a3a8bf42d81da036271d4fae5430197fa6effbaa0015795b08312acc05c" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954519-kIVflpyrAaKx8Iw43vS3", "proof_sha256": "e81987165d62db4cd013eb23b2d9ae5c387b668de57ff167b2ed9c9d985939fe" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954528-vWnEQKrAWYlaGWmnme3Z", "proof_sha256": "291057e328baadcae262d8d973d271c93588d547c77981b1badbf19ae17570c2" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954546-nVAqGZfRQQQKdWQluWMJ", "proof_sha256": "a0598fe994faf53cb0274ccf33262bdd13aa2e349b8b697343180d575e29f3c2" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954559-aDHSDIc5Z6AXyrOrCIPn", "proof_sha256": "499ae81c5900cb91c90427b7048634a4e3fa1ec07713d79d7fad855d006edcef" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954568-8kOGb0ANqLovyDbn6PTb", "proof_sha256": "ce0823d67e762996d0a3eac7ff018e06d8aa96d219ccea6fcbd3ba86b0aa6043" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954578-QzO49ej63TM4lrmcgk87", "proof_sha256": "decf90235e06883839a91f86523ace6d23088e82358a139112791497cc150e12" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954586-sm06npvHSy6nj7H0Ao9o", "proof_sha256": "c121d820a016f52aba91054ee72d1f7cdedd33df841e228af91ad5b51ec93ff5" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954593-L69JMjgyCAVgBV4QnwjB", "proof_sha256": "6d3962c3aadf5f66fc4f8f66bae3f4b2c3a1ae6892fa72334d08370cbfd199ca" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954601-e6XsUgT0U0DSCqoCpaOM", "proof_sha256": "1bc402771f276f7134efdaef8e3a8c507ee3edf8216335bcc0bdf8ef38cd58ac" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954609-HN3FYq9jzRwzQ29QCczA", "proof_sha256": "7b03a8cb6e3ab0709f0f6f64aa6d1716aa593d173cdeeffc0c121568dacffee5" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954618-YwstMpJlwEyQ1boOUxGe", "proof_sha256": "a5fa11498210570509c4c2b3be8ea8f1e9f97856a58292feb077d20efaedd723" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954626-Plz8jD132ZXi8Sg6aYdQ", "proof_sha256": "4ba5bb606593c126d145d59cb571b2d139d6242f3f5a242dea6d86f6d48b28bc" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954633-WT5rUteaOo0bHEDxvDwQ", "proof_sha256": "ecb761a4c0eb90e395963e342a9e6d46a552a7aa0e6e15156089a66721d267ee" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954641-1vcTvrVaU4TfJIyu4pvg", "proof_sha256": "75a110b5b43882df70ed630891737297c9713d460b41b7b42d06c01ce6cb596f" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954649-3qRAxbcLRSufJfW1Xqco", "proof_sha256": "a6932a7e63f4cb637a7b86b65d251e96b708365086cf7e8700742e4e0d6f4136" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954657-H63rp7TDOw4MAdcv6Bab", "proof_sha256": "7681b52e30668b3ce976339ba2a6c1d62636c1a9bf4fb21e53b6f1fb6b560ffb" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954665-XLnSmWhdN5ogBHV79rSp", "proof_sha256": "1ccb236f2d82db25714ad865c9dd306f0a0a4921959c3118a8765704d78bd4a9" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954673-zmhTNc9BVeMOtpuaFxS9", "proof_sha256": "d469007dccfe13d50352d84e60d7bc3f55c34eccc6c44a710b5a31f48ba938a1" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954682-SaapqfInnBdMqfgoeYhM", "proof_sha256": "a7d635eb73b312b6a9fa99242447c82b0ef01fe0606fc5c59506a2fa15f9c635" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954690-hFZA5C6myYcmf6NJfFnl", "proof_sha256": "a0d6e736a4404ea18e8851699e90e549fdc28b440d4028802c96fbc0b71f0d9e" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954700-nbkQ2LbzmHDAzeUqW1Jy", "proof_sha256": "db9c51856f093448a45f623cc14db1a43790fcad179fedb750aee83a77a51de8" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954707-MoyltWkl1Fxyx2x27dKM", "proof_sha256": "9f158918ec2b35aec27132136c5f513a2aa9349db299730caeae4804ff2b42ec" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954717-EdBmYcmOzina5hHuAZ3F", "proof_sha256": "f1147323d751a559d6b08357ffa1e55c600ee3f9adf18536d408cd3839d77e18" }, { "model": "thinkingmachines/inkling-small", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954725-2xgcv1ZRAGe0PGcGeNGr", "proof_sha256": "0cccb8d7d467836a9faa6e2413ef2b88bc708274102cbf4386dd5401c8aba17e" }, { "model": "thinkingmachines/inkling-small", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954734-inNumT6OlufXen9LO4iw", "proof_sha256": "3c153f29de22bd191808bc899ba8fa042bd53c50e5dae2e0a9226dce49320f01" }, { "model": "thinkingmachines/inkling-small", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954742-CZhrCbLcO0QVFZ5PJfws", "proof_sha256": "0e86408b77010390ec97a791d3f16394ba33a7dd608a86c9c675f43c2aa9e456" }, { "model": "thinkingmachines/inkling-small", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954750-du2GvgMiOYSgb1sRug2K", "proof_sha256": "52489d68aa0ecd7c6b32d11bf2b9c9a365cf94b3064368f817fbeb26663eba2c" }, { "model": "thinkingmachines/inkling-small", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954757-vAEtz6cQLtxtzeY4rYws", "proof_sha256": "67fa11fc8fd97cf4c2118a49e607e60142241db2fa71c4beee610540ae92f1b9" }, { "model": "thinkingmachines/inkling-small", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954765-lKyBfHtJt5fWh2pi6s1a", "proof_sha256": "3f60c1ab18928b72a7cf7868a19ad62e89ad38a789ae4fc0122328d427e772ef" }, { "model": "thinkingmachines/inkling-small", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954773-XFKuxxSBUFBu6CLSopHp", "proof_sha256": "8fb3c1c94afabff64912f0715a1b230bcb4f46a902b5a32e5a216859f17beed2" }, { "model": "thinkingmachines/inkling-small", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954780-HeJ6cHKGf6kSqFpdavZ6", "proof_sha256": "c84089395e6c80f7c9bb0851697568d1e09ca0ce8c09ddf1700983fdc23f9818" }, { "model": "thinkingmachines/inkling-small", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954789-NN6D9K5lTUKDImAeW9gU", "proof_sha256": "3cad1221580d3009aae2fe6dffc749f02402d98bfde01be125b46f365db553b5" }, { "model": "thinkingmachines/inkling-small", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954796-D2OWn0SlHSbW7Vurv97p", "proof_sha256": "85743d1231e0bef6c14d5b87181f98134394de72b6c48ca3a7b9a2246bd9802e" }, { "model": "thinkingmachines/inkling-small", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954804-n3hGPs4HpD8SfurJjqtq", "proof_sha256": "c389b37464c2c70c71b94f65cd3b0fc14c7cf1dd64c30d8b9c77777401cd8ac7" }, { "model": "thinkingmachines/inkling-small", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954813-fPLRYfViR9xV8IDT4vN6", "proof_sha256": "4cca7bcf266c5d0d05c23eebc55e84ac65104dd9d07c9fa75a6e5912b8ea7d5e" }, { "model": "thinkingmachines/inkling-small", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954820-hkjHm6mJHBbHj8OigOFM", "proof_sha256": "7849101bf4e8e0fccce72ff8a92afb12490350720cf0d0296a2556807d574c04" }, { "model": "thinkingmachines/inkling-small", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954828-sUCdelBabZTtzeFvwU37", "proof_sha256": "38e2b0804d18a8dd03e1b15266b98c339da39ee264b56ab94a34fb3bf23128ea" }, { "model": "thinkingmachines/inkling-small", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954836-ZEn164LJlA4T2a0BgtnJ", "proof_sha256": "458151eef22f84af38bf406fcd517ee23c351c75c899a355da2b92a1dbb7af16" }, { "model": "thinkingmachines/inkling-small", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954844-YJuVyqIzwiZzdHlZ8asx", "proof_sha256": "93cc11e7684b9d6efbe7716acdb815c968297295d6a67f87fa97a8aa224dc693" }, { "model": "thinkingmachines/inkling-small", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954852-tz7xry01NW2mT2IjAmj1", "proof_sha256": "ba2a5009a7474b5a371ab6f924bb60e0ea39623777e17396fed43fe474b13d8d" }, { "model": "thinkingmachines/inkling-small", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954860-69UYUBJjhQWNBUaytlhj", "proof_sha256": "ca6b13156c4580dcb3b87edac1b26ecaf7578e86ff929023e585c81226d79d01" }, { "model": "thinkingmachines/inkling-small", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954867-t5vn1jgqV2exQ63xN2ex", "proof_sha256": "db32a9652d3262fd4658093e236f7dc9d04de7ef04c1ab584c6812e9b9023a98" }, { "model": "thinkingmachines/inkling-small", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954875-vrpYVFBkVed3U7DXT1ZS", "proof_sha256": "fe9854f589d4fec6e5ff64daa87e4848ae7a07b869090bf164a5cd5a6ee127dc" }, { "model": "thinkingmachines/inkling-small", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954883-tNSPRKw9vpT7xqTA8tfG", "proof_sha256": "0feb63fcda9b5a48a7a6772b210f58b7648ad11cfb907d8fdf8c217c0519210c" }, { "model": "thinkingmachines/inkling-small", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954891-OJEeD8ezLqMp0jEKBQKE", "proof_sha256": "3c4b6247b19871e48f2bc08fff23d1aa944ab5836fc73c4258300bb228db6d7d" }, { "model": "thinkingmachines/inkling-small", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954898-DS3bxkh3KTAl6mDaBs1t", "proof_sha256": "00d099b41f9f014ec3b2ecde14d13a839846e1859a54037a2037ef1c3c12e0b3" }, { "model": "thinkingmachines/inkling-small", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954906-eUIiDL49KscrBUtmmtcz", "proof_sha256": "9ca2eedba977409116b3314eb8a96d78d16d9f351f69bc2b0b76a9c5443a5a17" }, { "model": "thinkingmachines/inkling-small", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954913-pkUzdCP9hpceJeZqYhOo", "proof_sha256": "c1fd10ee4d19ca9618c1b1f398e47ff1dc26d89656eb7d644a815d232da9c04a" }, { "model": "thinkingmachines/inkling-small", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954921-64nghhFbmCBdw3gSRHS9", "proof_sha256": "f74cf84a535c371174ffa3cbbfa183cfc84942d9ff4f5146b522c51e035e4da8" }, { "model": "thinkingmachines/inkling-small", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954929-L18IZnUFNbGfaLjJETpN", "proof_sha256": "93c96b713ea1c9442f017f20698bfeef8d8d510c883782f020ea493aa7f01a1d" }, { "model": "thinkingmachines/inkling-small", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954937-qcYpP44M9cf4WG1PmJSg", "proof_sha256": "1954d15e82f07d8d76a0b293fb2503da228eb73abde0fa52dafd377493c50d17" }, { "model": "thinkingmachines/inkling-small", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954945-xOyxSHRnnIkaHuzJJP00", "proof_sha256": "360f40d3407f7d85cad35e78497edd125167c3ffd2ec8875858876c3132847fd" }, { "model": "thinkingmachines/inkling-small", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954954-hu9emQvBI0h8XiKYP4aQ", "proof_sha256": "17b90abced120ff49bfc1468331e87b0259222567ae9ece3138853f729746fb7" }, { "model": "thinkingmachines/inkling-small", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954961-MR1w86D9FfKhLgBXz5Qq", "proof_sha256": "ef1922d29d9bc0bc3548a23f83d2a3a10de64af735554ddf5f113bbff017922a" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Near AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954256-JPODVU2vH0J8bqyyXVUe", "proof_sha256": "92d74acc79750f3cca4469f5277c560717da63c58d71c7583a7fe2fa188f5be6" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792954272-0ZQzSessnLxrrZSN3RfE", "proof_sha256": "befaad05ed357637d1045aa99fa31ab3906bbb9cdd2a0eee8c7cd4de67e32244" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Modal", "run_id": "20261025T200000Z", "generation_id": "gen-1792954302-XFd3MpbiZpTiEeFP5iPK", "proof_sha256": "10a61742cd9913eeb93c218419501ca694d69049bbbcbe2875499e59b6f006b2" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792954313-tRzTryNaU1GFvWFX0C2O", "proof_sha256": "b62e2711242c5214529298a1bfe888f02b3a13c41ad3a722e6ff506f33ab66c9" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792954368-OEBk1Lc25gaKLIBHYvJN", "proof_sha256": "246d5d3a9ae03920a74c661d756daa2c92b7c8d2fe7b74743eea869a3518beb4" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Reka", "run_id": "20261025T200000Z", "generation_id": "gen-1792954426-oszFWkukXBReuQCRRfOA", "proof_sha256": "da5b375b23e2f0881539b76e6deeee456fa8a4122d5889202d002942fa342e3b" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "ModelRun", "run_id": "20261025T200000Z", "generation_id": "gen-1792954448-D9Lru75PsjKvQNTwBznY", "proof_sha256": "10713e40a8cba9fcb858d807f7d1e367ccde07ba6534c998fc634d3f04e12791" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792954458-eOa4ZII0vndt2mfqUibu", "proof_sha256": "84665b79eb7f27e9eb85200e089cc63d1332ceead45e1de2895756d21fe7f505" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Near AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792954485-Bg3U1i6ImhGD2zJO748h", "proof_sha256": "367dcc4023c2743abddeb83adf1f6fa0eaed4960f0f2f1e0dd93012a88ec0ab6" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792954504-1BTUnkgWsqmIzchysVyA", "proof_sha256": "ab967a4278b4e405acbda48b52938714349befe99571939bf27be5a1fc903bf0" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "ModelRun", "run_id": "20261025T200000Z", "generation_id": "gen-1792954535-MhSd62LIYYx69NV49jXo", "proof_sha256": "72d434082518490ddb04fcf84642d876b02cb502b3c6b55e9b614d4ddad5806c" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Modal", "run_id": "20261025T200000Z", "generation_id": "gen-1792954548-lngmW2Oz2LpCTm4ZjjE7", "proof_sha256": "51be9a8a3cd102b3a540ebd3413ec954b26fa5acd64d22f9fd1597beec7477dd" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Reka", "run_id": "20261025T200000Z", "generation_id": "gen-1792954571-LNu1aOVxmPbeNkrwxVXF", "proof_sha256": "7ec93f3ebdeded0cb382465a904d5149ab385bb29285a54c86789bc80118171b" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Modal", "run_id": "20261025T200000Z", "generation_id": "gen-1792954584-i0X0IdMyEkFYa6i3W4dk", "proof_sha256": "eb14736d2cb18acd2aab19b58638f9abc2c953de0acb9066d6ff6b7820214dca" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Modal", "run_id": "20261025T200000Z", "generation_id": "gen-1792954597-l08sf9oK4WLxqo8QcSlH", "proof_sha256": "b711705be53d61e9aba79aa4698977e0ad641af6892c7b849d9b3ac6674dbfcf" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792954618-QTe1SP9JDwKuPan5jhhJ", "proof_sha256": "17861f4683b056aa7f3f47fbe988aa4888bcfbcce170a121dd9d177be6359e04" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792954654-OLBhOlRlIQOeyXhAAEiJ", "proof_sha256": "58a401adc14c2148f7d108c2be1f306a3ffdf7e120b4faef9218b59ac409ddf8" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "ModelRun", "run_id": "20261025T200000Z", "generation_id": "gen-1792954712-0EPCA3tlQlNuBhOXO8yg", "proof_sha256": "a13aa28fb0d2da1eb42d9acfb83fead509ef72442d883cf7a83f2674b51a4663" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792954727-vVm635YktsNkyXhucyY1", "proof_sha256": "5dac1571eb1dc66e0122a63efa90895cb075472355e70450f4bd3914e76d87f2" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mancer 2", "run_id": "20261025T200000Z", "generation_id": "gen-1792954757-HoYrQXFy2Y8uAe6Kvaop", "proof_sha256": "7bac82cf02f760964a18c23a1b2469c3f6ee008879f2a2b22a7caeda84dd1654" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792954771-0lAVvg1z2bvci0HiueTe", "proof_sha256": "6f091049717f805f0acba6530ff389121fee1ead419656ae24127f9cd3267e85" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "ModelRun", "run_id": "20261025T200000Z", "generation_id": "gen-1792954790-6K2MimDatNM0X5WPwaU2", "proof_sha256": "438fd3d4bd5900fa0f84c9e1517b577a1b50a57016e5ada96a265cdee97663fc" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792954806-2hm7fHKZPZnvQf9ItOhv", "proof_sha256": "13386e5f765f9224a2f491ab23a736c768c14cbbc11293aa0261a9fcf1f5a750" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Modal", "run_id": "20261025T200000Z", "generation_id": "gen-1792954839-J9wYmbPuhDz31QBjbgXK", "proof_sha256": "528bfeec9b677a6e0335cd1d955d980e1065367008b4f35357563e8d796feef7" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "ModelRun", "run_id": "20261025T200000Z", "generation_id": "gen-1792954852-qFtkk5vynnmiEV4DwzA1", "proof_sha256": "0683af99a0215b04850692ed9a04d19bba46702fdf0d04cbe976aecd5e63a578" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DekaLLM", "run_id": "20261025T200000Z", "generation_id": "gen-1792954870-tKXQUoRXHC8abm5zKkcR", "proof_sha256": "cd2c696d338428ff66ce1128c1f1003475613da38fc336e87cd0266ee6c22027" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792954923-BNpJpvnp50EkedikZM1R", "proof_sha256": "8dc7f7e749f3667171b1aea1f8324bb3499f80db4cf6ce760217fee1055b1912" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792954948-vjjSy8DyD38mv0UYHeVp", "proof_sha256": "3f92257f67f508454d5bdff97efe131ec44037f85a013bce312a01a87dae03c1" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792955004-uZq6b9Y0KMFeXknzcOId", "proof_sha256": "e515fbfc602d479c9d1c940c9c51f0987011f6e270b0328ccd61f4f8ee525ded" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "ModelRun", "run_id": "20261025T200000Z", "generation_id": "gen-1792955064-gnOGqqyu8JbCCvfSZcnz", "proof_sha256": "321131188e4601b057f565114bddef2fe89a1b814105b64fbe6647f3de7b406d" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792955082-DXOSHIfJ0cH8eRoHqx8f", "proof_sha256": "18d9731aac1fd3b83f51db85961f65442852345f44503797d4815ec0e1ac0cf2" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792955113-KBkK3EnAjE22R39u3ZJL", "proof_sha256": "bb1921e3b568a1c62492fe524009fe28da5c14f484dbcfe4a2e8015ed3b2d913" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "ModelRun", "run_id": "20261025T200000Z", "generation_id": "gen-1792955143-8alLLBCpe0kdMOyMTs9q", "proof_sha256": "5eff0c13de027347edd15b990a424709b5a4eac19169d234812d8351d477523d" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Modal", "run_id": "20261025T200000Z", "generation_id": "gen-1792955157-S6gCAMc6ooDEStRan4Zp", "proof_sha256": "14efabada7ef29ed99c3cc8d7080a95a5d8abe53c86b4aa84d477b2e252f692e" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Modal", "run_id": "20261025T200000Z", "generation_id": "gen-1792955179-3ON2fkDdG4PwsNpDhOQX", "proof_sha256": "4886eca803d1f9e2d7c49249813b3a5a3a51cbbb880e272aa1d8fb6d22312508" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "ModelRun", "run_id": "20261025T200000Z", "generation_id": "gen-1792955193-bytrJQ3ajRxvUT1ejHwK", "proof_sha256": "b1fbe62d9c281b74e41515f1c4dd8013a6accac0389354646c922d822471d05a" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "ModelRun", "run_id": "20261025T200000Z", "generation_id": "gen-1792955211-3Is3sQ2CLXv85DL0bknH", "proof_sha256": "5dab791edbd7aa941ea3ca37dcd1e78f70ceedff5ce312bc9a2cf6d42f326f7c" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Modal", "run_id": "20261025T200000Z", "generation_id": "gen-1792955224-HAHDfa2QVLnlUq9zDK32", "proof_sha256": "b0623ff9bfad4d9b6d7c8a37f46054b215f0b628b17333b2da926beae1475080" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DekaLLM", "run_id": "20261025T200000Z", "generation_id": "gen-1792955242-3AIGAefr7ZEc6grVKGLy", "proof_sha256": "78525e81136a95d1898ca64f1ba466978afa207b604e6158e20cc54830ff90b6" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Modal", "run_id": "20261025T200000Z", "generation_id": "gen-1792955257-qEhGDsQmX10TQXfVSjdS", "proof_sha256": "975e736f522a463c990876c2153cfb98131fd2cdd1e4d1e65d7be19f38f2328c" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792955272-iMLeCIiZU0tBHrbUWAff", "proof_sha256": "00f3076489e8943758897da8349658db5c980d723d411de4ed8eab3724983c3b" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Modal", "run_id": "20261025T200000Z", "generation_id": "gen-1792955292-K3WOlgDKOc4kXbFnLIBs", "proof_sha256": "0b8736ecda119705651840f57d6bf12cd122fbf58db3ec7acf06c180e78c0d33" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Reka", "run_id": "20261025T200000Z", "generation_id": "gen-1792955318-SP39JFf8bv5rpuoQPDal", "proof_sha256": "bb56094d6dfb584d3fa4126f44630c52a3590493bce5a9b894bb8c9124090d86" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792955334-m4ApX5TeED5Vkfu3U1qt", "proof_sha256": "c49d6cd3a6fd4b13ce936298b608a1b868fef891b1a31335c4685254f3ea03e6" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792955412-eN7Hd5ZDpVaUuPM539aU", "proof_sha256": "a412f50b44af016f38ecb6a688c43c5fe93062a6b44c0dd55d3ba3f3bcdb0411" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Modal", "run_id": "20261025T200000Z", "generation_id": "gen-1792955457-zYoU32ngCrfL45jYRkGl", "proof_sha256": "a5994dbdaf11dffd3acdb17846f56e56953234a7bc1943a6a3f79a0a61dc23c3" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DekaLLM", "run_id": "20261025T200000Z", "generation_id": "gen-1792955501-eZzIKowrZGkjUkoxVj4Y", "proof_sha256": "1a125d4a44cefef97c9f581b7910d5d3511a1de1c9c927f0708856a4f6e1ee64" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792955547-s8jjVwZ6J9sJ560W4VOP", "proof_sha256": "72e66bbcb377116a7b64228ddbbd2ba468cdf0b7422a1e561cfd5bfff0420bf2" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "ModelRun", "run_id": "20261025T200000Z", "generation_id": "gen-1792955609-FB8lW4EbKhbfifH7jrmj", "proof_sha256": "681ac940deb4cfb5c9cbcfd9f9f8304181187ba9819b5bf5193092f427c8b830" }, { "model": "qwen/qwen3.8-27b", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792955640-4JRBOOhuIKxVpr7RtBAg", "proof_sha256": "360e23633869961766056933d323f31243219ea923f71329bb07f3a96552a118" }, { "model": "qwen/qwen3.8-27b", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792955678-lEFP2YAg8pgSn3ohVAo3", "proof_sha256": "b7e7c46b18ad624885b8cd4a064eb8bb4e51c6b8d19a18a0e6647ebc995d69cf" }, { "model": "qwen/qwen3.8-27b", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Near AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792955742-49oCSVnqOJcUywr9VwGL", "proof_sha256": "f36f7eb63e06b0a15c4222845ec86aba5b2d9c4d913d8c37a5797c07993e7da1" }, { "model": "qwen/qwen3.8-27b", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792955797-3aKpyN3lBt6Rc59FQ05M", "proof_sha256": "1d8b803bf3f924f38c6a368ef1d913371257bbbacc0f10d85f1e89998b311325" }, { "model": "qwen/qwen3.8-27b", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955850-dQsj6gHSpKxgMJ1a9npN", "proof_sha256": "8bd4723ea0a49dac35d59b7ef3930a37dab339fe1306d1da0c8d1d860e32b0ad" }, { "model": "qwen/qwen3.8-27b", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955930-vaFwWjxNEDovsSPHUsWY", "proof_sha256": "22e2b44c971560bccab21caa3e45732b19e2fe446ca36f8fe53d102a0ea782be" }, { "model": "qwen/qwen3.8-27b", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792955986-D0lmGSttA5vxwYJGM6iu", "proof_sha256": "4b3e6e6130b873166c8e927ac6144507ebd578bd9966aafc94147320b08bdde1" }, { "model": "qwen/qwen3.8-27b", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Modal", "run_id": "20261025T200000Z", "generation_id": "gen-1792956006-xnKANWb8lHmhJPpWfa0M", "proof_sha256": "2c390ed2cd8872fbb30522c5b3c7130d42539f377305f4e166470540e559a9bf" }, { "model": "qwen/qwen3.8-27b", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Near AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792956017-aFl6M96xRc02Sbr9rvGJ", "proof_sha256": "71ec595e92a43d025ab748d1282ca9cd76491917349dd41c3b29c80376d31553" }, { "model": "qwen/qwen3.8-27b", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792956032-EbOlZzcPkpR0OYQzLD4q", "proof_sha256": "d70e4bcfeeff6a773f4b019e3f0c3a9bc2d25c2bb1e9cdb9f63e05f7c25f66bc" }, { "model": "qwen/qwen3.8-27b", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792956047-4ykmQQC1j5LJqyu7hmdd", "proof_sha256": "a4e45a411c05e90264280413cf8313df3cbe8123cc972ece10831d7d1afcb6b5" }, { "model": "qwen/qwen3.8-27b", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Reka", "run_id": "20261025T200000Z", "generation_id": "gen-1792956065-4uPvkjGSgRUs8jOrSnkB", "proof_sha256": "6619a23c6d972db39d2859caa4734706f7dcad9fb3c495f87113ecd514251458" }, { "model": "qwen/qwen3.8-27b", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Modal", "run_id": "20261025T200000Z", "generation_id": "gen-1792956082-BJQAcMr6qiKHAMLjFzrh", "proof_sha256": "c72301ddbde2b420a855b28503337874a66fdef0874a2b3404be21a05ff18c16" }, { "model": "qwen/qwen3.8-27b", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956099-MEoFCblkwzDJTvjFFIVF", "proof_sha256": "be1911cc1316cf26ca44eb633859c113bd61d55d1625c0c52fd7a011e34cbfd6" }, { "model": "qwen/qwen3.8-27b", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Reka", "run_id": "20261025T200000Z", "generation_id": "gen-1792956124-bpqO5QiWcIvpokjh3Vna", "proof_sha256": "61789b63b3778373d707c547bec57aec874a892aec3676d1b68684da02504af5" }, { "model": "qwen/qwen3.8-27b", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "ModelRun", "run_id": "20261025T200000Z", "generation_id": "gen-1792956137-rcyQoNrjSnrcSAVmku4c", "proof_sha256": "57b1fb5f70134f0f7a7b1ada8037f7fd1ba960c5a92b8938fe9f510892787ee2" }, { "model": "qwen/qwen3.8-27b", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T200000Z", "generation_id": "gen-1792956146-cIadHOSWP6iHB3d44p78", "proof_sha256": "fb44a086bd4fba14b0552ae9fd7eb3838587a60139d33149c39d0410fb1dfc39" }, { "model": "qwen/qwen3.8-27b", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Near AI", "run_id": "20261025T200000Z", "generation_id": "gen-1792956157-RUMfntgHMN6CwW3fF5QH", "proof_sha256": "282d5953503b9c0b46b2c3cabe18061b9b01e6334b14a4bb7466c55c84f24f30" }, { "model": "qwen/qwen3.8-27b", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792956167-y2N61LNfAAjKSh0Gy85I", "proof_sha256": "194e0f5b9e8a6b6efe096841239c634d683fe20dc4a4bbfcdd30c90c6b971953" }, { "model": "qwen/qwen3.8-27b", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Reka", "run_id": "20261025T200000Z", "generation_id": "gen-1792956211-pzA84VaiXTnrt3aXphxp", "proof_sha256": "4fc8a21b672d8222bcb5bd88a29b0d5dceb0f0412130f67d27a1b943e5ae1caa" }, { "model": "qwen/qwen3.8-27b", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Modal", "run_id": "20261025T200000Z", "generation_id": "gen-1792956225-YVlAoHB4xQqwgGqXSD4Q", "proof_sha256": "e0ea34793121bf231b373d7fbcb827b9668ea83b9383f0e4037c0ba047d53a3a" }, { "model": "qwen/qwen3.8-27b", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792956234-ORQOE8ZmSEIUSIIbE2cj", "proof_sha256": "e4a7bddca9181c4da059822f68382d3bf67cd68987a760d1a01b36f1f87b9bca" }, { "model": "qwen/qwen3.8-27b", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "| total | ≈ 3", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956251-qUL2zmmjVZoPEuQ8enH6", "proof_sha256": "e5a06be0c7607948975e52f3485390f1b3ec6b60d2a291c0671756dde07c50e2" }, { "model": "qwen/qwen3.8-27b", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Modal", "run_id": "20261025T200000Z", "generation_id": "gen-1792956300-4dSSjsge1BfsKozR7Ky2", "proof_sha256": "3a297bdc8f92ae581b8daab74dc062a1510b995dabf3d71da9232d4323d3f69d" }, { "model": "qwen/qwen3.8-27b", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "ModelRun", "run_id": "20261025T200000Z", "generation_id": "gen-1792956309-ullfFGuvxKqAqcIgnFVZ", "proof_sha256": "ed93976a3266bcc06c03ed2fbb1c35576ae65c4db2d3eed94d16521e9e0c9c9a" }, { "model": "qwen/qwen3.8-27b", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Reka", "run_id": "20261025T200000Z", "generation_id": "gen-1792956321-SE0gJ6IIm4EZDopkEBYe", "proof_sha256": "42d2bb4f1d539ada57fe74159412f1a322405b8e0e7cfdfe168054644c411208" }, { "model": "qwen/qwen3.8-27b", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "ModelRun", "run_id": "20261025T200000Z", "generation_id": "gen-1792956337-RxLLtgyxyzmSaFc5BExU", "proof_sha256": "fd194dee6c4df3a3155755cf92734c1ac0c41ff32f13bf5b3a2b796685b64b73" }, { "model": "qwen/qwen3.8-27b", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Parasail", "run_id": "20261025T200000Z", "generation_id": "gen-1792956357-A8sF1g5vuJ4qem4GRe1T", "proof_sha256": "66aa0944be3c56ca3595e93247f2df1c13abe5acafa93e23607843faec89fec8" }, { "model": "qwen/qwen3.8-27b", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Reka", "run_id": "20261025T200000Z", "generation_id": "gen-1792956405-Nrm0NcDuzzQntwnfU66P", "proof_sha256": "b2da1dfdb295f7a201f82a05ae9eb0c68dae268779135484750795422493dc23" }, { "model": "qwen/qwen3.8-27b", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Reka", "run_id": "20261025T200000Z", "generation_id": "gen-1792956421-2TYgZ0YXI2Ed3swelLPU", "proof_sha256": "e7fd18ed21bc391512242724ae711d5076800571bf9b3449724f969ca992c286" }, { "model": "qwen/qwen3.8-27b", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T200000Z", "generation_id": "gen-1792956436-AOIdb9vcmFd0AvHRIhCm", "proof_sha256": "39d040b82ae2e73f05362114f5ccbdbf33cd04979b5cf6b7952bf2f23c0c09c9" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954256-ND1tjYSMhfBW1lwXcCo9", "proof_sha256": "51e72fddc9f14b02111355ff8f8a42efdcb5653923b350874d3efd2677ca2b7d" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954269-TDiT0ToTZmcQNIg3goOu", "proof_sha256": "afa2a78052b50e2864adc56785a206f1245f5c7494ff85e6121b956f6d6519fb" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954283-D3LJfvuJDGceCWRDZBXT", "proof_sha256": "6a670c7b2d61f5209ce93b8a0bb481b33e2815d989dd5ca3edbb31ab70fd22ee" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954297-bCopYtedRdsYTfmCv1hQ", "proof_sha256": "fcacca1963725dfaa9bcadde1b797e80bad0ee0ab8a07764f4c1025f2b9f9c55" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "$= 1993 d + 1993 \\times 996 d = 1993 d (1 + 99", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954309-rG4sqjVWmTtmwvQFx3vJ", "proof_sha256": "c9164346b778cf0c5d37d91d3eeee902bd02501e6bbb43ec6b285adcb5f8a73e" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954325-1ZGJC3Y869DzMqWTbLx0", "proof_sha256": "66a7d5e1c476cf1399062e42088377647a0e9809d05611528c5ebf3e2683a24d" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954337-VuRgAeLQ0ynsCznkNTk4", "proof_sha256": "f660b02ad51dc9afa9add8c7f5364a193400563bfcee3018e813d01743bd048a" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954347-5TR2XKVHikQ6dgbm46Fd", "proof_sha256": "75e5bd52d34c4e0f18b9ae00ef8c490d8cf037263f94d2c02a0905c68dcf7c45" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954356-eeVfwbqyJdbXQQe4tAtH", "proof_sha256": "61838bf6bfaf64be7bf4ed2439f669420d13e35e258e53cf5046fce553eb216d" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954368-3CGS2l8mB0azVNS35Bj3", "proof_sha256": "9c98c22346edb627d73ddff3f0729f1fbfd40de28ec380ec2ef1b96720ad4e58" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954385-fSx9mRNZd2JUYVDQY4P9", "proof_sha256": "7e4a753d66cec3442033dfa16c189ca2d62f398f27c76b71188a5132c1cadb9d" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954396-JseubmRd8Z6kQESy5RAg", "proof_sha256": "c4d3078decbec8742c59370f3159ab8871ae4d24afd95f0055d8ac6fbf775e4b" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954406-peAH83qEY1hvOV0yiU8F", "proof_sha256": "497f8f7911b2bed3e4a10168f93be318acf36db3264a69a97ad81cb8e3a84b89" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954415-h9h15lBM4y281lNYeNqM", "proof_sha256": "bc5811892aba001824fbaea98b412e5474e9c18802ef30fa760d5fdd33ffd60e" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954427-Exht1Yo42QaOAmTviUjA", "proof_sha256": "b4504576c7d7bd9a46036f124ceb244818738c60dad97aa3bdbca0dfccd1bb28" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954440-vm77G630mNtRPsTlsGMK", "proof_sha256": "79851698c551243e60b2eb24617608987b7966e155e57f078d37435153f9a37f" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954452-f5XY772I0LTvL9g8nInf", "proof_sha256": "d6b5a8704cd62196b9e52e53062ad156c28f27950a49d7de8d9fdab87b012e3a" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954460-0BdFGL6Ni2XCxWDwy2XA", "proof_sha256": "0db4ecf73b3cc4fc43ad8ba15981dbfcfd4a30677a5e4eb336f969f8b89e6be9" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954470-UuODGNCbwQ8mqdvyeyfG", "proof_sha256": "02311bab4e584a311a0c8356f80dc470f52ed4ed67da7162b2e1b7ad9cfb1ee2" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954481-xELh3VLPaVZBGqRDwiQw", "proof_sha256": "facc685a52970dd9cd4dbc6f6afbc81de3a3e5230262f418c9cda66f2753087d" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954491-ElFSTQAzGycvbDAXO80i", "proof_sha256": "7638ee071a28f0edae328e561b8607c7619ed060ea27d433c82019c3a1cbdef2" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "$$\\text{slot } 0, \\text{ a}, \\text{ slot } 1, \\text{ a}, \\text{ slot } 2, \\text{ a}, \\text{ slot } 3, \\text{ a}, \\text{ slot } 4, \\text{ a}, \\text{", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954501-UwjbauzE8BTNvEM3otqv", "proof_sha256": "5f3551ac7b75787bfa80249a7dcf28d9eff6dc1d4f1c860f9ed097d52200094b" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954521-h9DHqMcNvJZhey5LHvDi", "proof_sha256": "3a357ba0d458ee1e92de8de0a789b1c4ba6e489514185ae6dad0f4c669cdb16a" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954533-EppTeJ2RLy8QjGuKHrEu", "proof_sha256": "98e24b0265b014f72e0a8ce38b35b3f055f36b4b7e9d89f11b8190402b76de34" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954544-MJC7vwMOsMudNy36np7e", "proof_sha256": "f618c66a1fb148e39cd95d617412778b3eb569c19efd1ff6682bdbb0004cbb09" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954558-72XiiGSNrEkdWhATaEdN", "proof_sha256": "3494a90fcaeb19915c0d360b149494ba60c4b703fda9ff35665b1dc414bf2db6" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954567-aMS9UTTG2qKdMJwAaWjE", "proof_sha256": "93ddcd745fc4e54a1f0db792b8d4613aaf627fd91be9c8bd405fe6cc24a55602" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954580-j0Uskcz2UlmQpTymYMEV", "proof_sha256": "88a42622107e6f173df6d866c96748d37a7d7babc1ba9741f72fb531c51339fb" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954592-i3zcKDNsoJbiToXLCJrS", "proof_sha256": "546e118b80dfc69519e1476bc754a5bd3761e7adc025ee6f7648635fe76937d1" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954608-czCb9w4x5m8SUYFYTimy", "proof_sha256": "c0cec298874638010e46383224389ad18e5efa889ed3064efd33b07f612a72c7" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954621-bqCTTGapQuK60J7YwLMY", "proof_sha256": "59338b3c56e7fbd256c279642b57da05138722f66cf3605febb0927682377d00" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954634-skhTfqabguyG3N3Dsq0J", "proof_sha256": "e5c32eb62c0fe2628889db5cbad5a983f6964228f035f7472eb8996bfba59adc" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954647-oRoPY6qEzjRPb5Ximdtt", "proof_sha256": "5661091781e58adb5fbb919f9ce8dc6e079d203a73b351521a312c87aa4c97e2" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954661-vLAPJl7WI7X6GZ3zU1mc", "proof_sha256": "e9bd5661a551b59d278e98e4464e0e2082866d4ea621f00d3ea32d28ef96c981" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954674-i1BjPYGKroADfwcYDXYh", "proof_sha256": "331001821fe822a7d04fdeda47067b1fbc5a43f9463718e532d0a68b94390db6" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954683-QrrIIW8N5DyQ7OJutOjt", "proof_sha256": "371e5f741c1f9f6607adbf559c73b4f2ec6ab06a888d8a5902663fb14e065455" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954695-h9w98mjjghcxAOcAj615", "proof_sha256": "f428ff67430063ed22dc817fd9067c1d53be48b9ef00c8fc0c0d0a3fecb78d40" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954707-gDRQUqaBf1khUxsIOnOz", "proof_sha256": "ff6993ee7b0695f171a9baf1c3e1ae27029a3ef561cd34fd92d1c5c816b7e095" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954716-jYzJNkgvuDvBjm0p0ozr", "proof_sha256": "c01b56967d5ca44d96de5c30606900d4d41bfbe91502fdce85ce8164757c7e03" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954727-LNcZI0bgyq7PnLLVq7ED", "proof_sha256": "2430c659db2a73ef6499dd2a6eb20f11e9922d2058a0bc1761bdc33502180eae" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954737-I4pTnsYRDNyGhYRoiw7i", "proof_sha256": "a92ab2440f84683dd20dc3bd1c31b32289988c76ec99b9dabdc3b28f8edfde33" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954747-eet45l8kZqqk9Fw3Vo9x", "proof_sha256": "ecb7e2d0cba4b44a6736607b03693ace9a216798178aaf8b130149fd6a66f0f1" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954756-hOkKzyxzS23RV7FneO0f", "proof_sha256": "47e5a0c0e75dd6c3c2043ff05b405966cbcb833f8235ad171514219418ac1938" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "$$\\text{dpm} = \\frac{123 \\text{ cpm}}{0.80} = 153.75 \\text", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954765-SEzc3hjcmocI1iYK6nuC", "proof_sha256": "2a11878217897c5c5e8e87c0bed4fd0bea22e6dd0536775b76b3eda20c671ad4" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954782-RNsDI9HZWoUYge7ISckS", "proof_sha256": "627450b9e5e0efe9541a0bba83b7a2a4af8dd8f65e65e2b1ad0d4e89c66f16a3" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954793-3cpHIKeL9rSztRULdfM3", "proof_sha256": "3970cd152cda106f136db9bd3b3199922dc06fb21b7914876352a9c8d4df6724" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954805-YEHNBaROhjcYH5N5aYKa", "proof_sha256": "4f0eb7b0b63a0153ac81823a062fc9682c5885720089660c9bc7616527d2e535" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954814-AI69O54Xx0daF8LBXCT4", "proof_sha256": "abc2567c46031f1eeb999e4b55f50b87e2da3b0f05fc5c606677a7c8c1fabf6a" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954827-EKjnOKU2vjF3fsMrQTHb", "proof_sha256": "d73889e62029242de8931d53e7d5120e775cd4573436fa760c8421283a23f135" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954837-tHYgAmWtupIrfsbkuqUS", "proof_sha256": "5ab6beb33d00a70f416fef4efa3475d03395025ab6934ced0c19c19b622f9269" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954846-ewFU9PIVilbdGSHiywrz", "proof_sha256": "a50010a1939fb030659afa566139cd9f406c7ff735aa6eac03366e793e327cda" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954856-FYXsi4PsYHrXCeO0NJUx", "proof_sha256": "08137bd8d4ca3e451d223a7a61090385dcbc9140c395ffbd959245545e890a76" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954867-fbheXdPzFQFWeA9HAD00", "proof_sha256": "abb66778a4470cfe39b05acd936354bf5b3abc1e949e1882f71a2792b1febe8e" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954876-5bvITjiTOBiEpqJr1GeG", "proof_sha256": "92c11ecac2c1900f36c93b7b1d8f2b889402b90c790e4bd1595c6f06f6ff58e7" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954886-uN8UItod3mbFsvPwXALq", "proof_sha256": "a5865df64159af54b5aa708936fd9f2fb6fa8a226e80d61ce419e704d4cbfc95" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954896-hdTrwZ3A29wisGAVYNNS", "proof_sha256": "61ce5c42150f6be629c6d6c75c3c05f0a4224c607469af8b48e302f8e6edb7f6" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954906-rXgyhAMapGnTqckKaQMj", "proof_sha256": "90a7d5bf6b5aebe1932d53d2da664c3aad6773d9ed4096cb1005696fc52a45fa" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954917-o6t0n4HtnVcaWx3nxfTh", "proof_sha256": "b3caffe360be34664f531d248737c581ac1b509e50a63521e3ca60d536fff08d" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954925-046BSCi0bFYy8wRn6xlU", "proof_sha256": "d2e74cf9fd3f6064adb06850735cc58d893a37c854848727e5be301915f763a2" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954934-EhYRojauy9sGb8wpFB6v", "proof_sha256": "dd4c7940d4a5a26d401435e7eda625a756dabb664b7dc6d7edd65255e8fe3915" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954941-RI6s7XLJeVaPfHKFe7oF", "proof_sha256": "7346189ce2dadd9da0a530e5d4a58696fceeb439fb80d161f0c40175bc019314" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954950-d3AtSZfEzh87yQESPCHM", "proof_sha256": "e2b3a562bf3238114ea22bf5074e236b29ffbd88dca7442e6e5a0ca8647197c3" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954958-NRYtZfMzEXoQgsgqiqgh", "proof_sha256": "f90d7a748c7dda8be64ec287bd260859ce3a834ead525b0942dab8caf38469e8" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954967-edeLf6sd8hkfMvcxlhR4", "proof_sha256": "ae261a27d6f21a443f1082b251a3fdf3c82adbd45663e7b67d5aeb1f62ff4905" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954976-OtuH3mUwma05mlzQUKjy", "proof_sha256": "e4aedc513f36b4bc98749182fd931f9ab83b112e3d37ae627972ffde9a457bd6" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954984-MMIt0F4AQsKojTRNI8HQ", "proof_sha256": "c0e60b06a3c31dc3ebb877bd5452265f619b959f062f213e8750bad82e34141e" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792954992-M780TaWniDoCvICwBcDh", "proof_sha256": "7d6f747d7ce4ea9ce2c0364563a8b75dd49705b4ea539393a5364a6873e513d4" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792955000-fWysxXSqO27dap0fmwCI", "proof_sha256": "d237e6c21169b85e19a9906b6b1a29fba578224f67a709b45a76e0f72588409b" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792955009-SDkTOqqGflOeG7N4nk3Q", "proof_sha256": "996f6fab25608cb350ea3d39a389bc971399517ce27d4865d58b33ff11611f70" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792955016-0ycRkV5G2MLpIFQnsrFi", "proof_sha256": "e24935f416c4d8ea783685519813f2ff698127ddab85295735aacf172eb3a0c2" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792955024-WpSPDOV3vycadn5WaCSZ", "proof_sha256": "325a059bad2547d11b1c2c827e0eb76ebe69e3578b800d9e55f2fe17fb867df8" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792955032-QLuHy9OE4iTjdqblmfXT", "proof_sha256": "a987e6081b6e55d359ab4075dda7f3c9f0296bc52b6870706525dfd4b83fc7f1" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792955047-IsEmWKDMcvtIXxJjBNtM", "proof_sha256": "f730fe361adc9173ad4df55553f36f61fce2060354c34aa7397475a375bcce72" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792955056-7LNNRfD6p19ekmOwbc2z", "proof_sha256": "72486d8b21aef503098919cb91681b56ac827983546ab83c92521a47e8d19fad" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792955068-WigmDL3Q5HLDAAEZNjlo", "proof_sha256": "0cdb49ab19cfa31ceef3a07d091ef3e544fe8c83a7192e06f6eca1500b8edb84" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792955077-BXTbHF9CY7Nj8fgZ8Z90", "proof_sha256": "f3d119b6f241e52e15f76250b9736858e59261fd131adc967ef9b799a96a4422" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792955097-4QtpCQvr8SfFZj7iTiBD", "proof_sha256": "7e644d236d74e99ae3171ab2ad4aa82d247fe968410b36e025fcd6a438013d36" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792955108-1aqC1gcRKbkKx32XWOE7", "proof_sha256": "8dd9b53f5d51cdc22ff4499c75d33e6d48b1d2f6e37d8d4ff9f59aa4a67b9445" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792955116-R9KivDqAuwfkWT7eWgDl", "proof_sha256": "cea21a48a7fa5d8bdf169008937058d8a576ec298cd4a8a4cb4b82298df2f772" }, { "model": "google/gemini-3.5-flash-lite", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T200000Z", "generation_id": "gen-1792955123-4wHFo6bzJnG2oHwQIx8b", "proof_sha256": "5766393df7aa8323e429a60df018f37963d0520b9327c5b402d88dbc364732c0" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261025T200000Z", "generation_id": "gen-1792954256-X5TYxkdYcDliWcU8Pu2Z", "proof_sha256": "8f54a7f5286079e355e9ae4aaea9d6a36f5a3d5a734f643a598141d2e030c177" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261025T200000Z", "generation_id": "gen-1792954267-7aPVxPLnxGhrxekmIRSz", "proof_sha256": "f054017b0bdb115653bdc4d2223eca5f8e80a8d006af852db294c9a8268b26e2" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261025T200000Z", "generation_id": "gen-1792954286-QP8I5D0FewPpEqq6tZaU", "proof_sha256": "910c8093fa0fdddb6e6ff500572859788533493170a32d2a3e8bbc94b5abd0b6" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261025T200000Z", "generation_id": "gen-1792954298-JIX4r0zWNRqQ1ZzhV4gM", "proof_sha256": "419d58efa7b59bf16760d71f78dd3bb2add9e58db9f79f5316a68014308ca530" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261025T200000Z", "generation_id": "gen-1792954313-PkGbEKyg29ivfMq0yVQi", "proof_sha256": "a68042d3ccfcfd06fe9b59ee5852258cb5bcdd056cd535464243f9c5b45d9230" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261025T200000Z", "generation_id": "gen-1792954332-8y7l5ADrScHbfWSD1446", "proof_sha256": "9523f771f13b0483668daedbde61214d6929473a2e2722712ff0044d6db889de" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261025T200000Z", "generation_id": "gen-1792954347-2KILx5pqzhyauptPTFxP", "proof_sha256": "06a21584d091f555e679129b26d71dab834ebbf8a48809d7093533d309e67899" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261025T200000Z", "generation_id": "gen-1792954358-9Ssl0IPyuqlRTo3mFPX6", "proof_sha256": "34481514d2a711f930d42a9624806052de492caca146abb2d78b0cbf5e56178d" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261025T200000Z", "generation_id": "gen-1792954371-HTrKHb61t6qrO969F8AL", "proof_sha256": "a9c1d2107cb33133142de2a17701d269d96e307d63c41f7f2d014e640626e9b9" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261025T200000Z", "generation_id": "gen-1792954383-3yjKNmbZo9CrNNoZf1kn", "proof_sha256": "fbb94f966c710cfb94f478f6727cff149347207657db1141de802a03a97590f3" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261025T200000Z", "generation_id": "gen-1792954398-68IbstPkrHimfp2vOGzk", "proof_sha256": "fe978a5a77a6f5e5a49fabac587736fcaccfa55a7dc0959ac1de665e44d1c2ed" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261025T200000Z", "generation_id": "gen-1792954410-M7AIaT7XLPtLpqw7RO8H", "proof_sha256": "1f5449aafe9aaf7dbd45f2d3545c0a9bc2cc0f59af2c0505cda9a835a52eddee" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954428-rr3lkc68hGg9bjMXuyE6", "proof_sha256": "79a87b2cf58d82d566619f80c793bd60af1b96db153430a81d0f97816e05e79e" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261025T200000Z", "generation_id": "gen-1792954438-fuAlrpPQUORpglLP94Oc", "proof_sha256": "eb98994f440541915a2655201d37e3e70888254909ad84cd0d3bc55e52787c37" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i}}", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261025T200000Z", "generation_id": "gen-1792954452-14CNtRpyxmw6nPnSWOOX", "proof_sha256": "86169eea86c478a10cb21f20f8b061dbcde65f8d5bd1ceda9699e1a55d41d261" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954466-FmAK9UvR3LA7PXO0HKLa", "proof_sha256": "9a44c6f5ad42daceffb9808778864687680b9b17814ffb85922dc63bd90c5bae" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j}}", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261025T200000Z", "generation_id": "gen-1792954481-yHlmbZrlkSOiUhdZ6hna", "proof_sha256": "a467ba0e7b96007a8e9094c25b80d0004d33d3a7032a4a95892085c0db749570" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261025T200000Z", "generation_id": "gen-1792954497-pgSVsHNQV9J1tAIw8Msy", "proof_sha256": "79846c0f0c492c748c10c8189272158ccc47f1f0c4d4cab7a475891f079d8dd1" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954515-L0UaI63tfkPFL4Ca01T1", "proof_sha256": "f384235f2169218b76ab10110532718dc2bec5e10c69471199e5e96bfd50d981" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954528-Ry7WaQKtP1dDoV2VVcYe", "proof_sha256": "3288493ae697d49ae9f062aecafe664eeb34fb5384588176dddf5df12766c382" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954547-TNyNemcVcnuUAQyqmsUn", "proof_sha256": "f7f7944caffab6db0f3490d10df5fb18fc7faa495758e7bdf8e123aac66b663c" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792954556-mALBPx3bFpsOVCZ0ZOTr", "proof_sha256": "aef0198c41ca3bae562991c36ecf0a24d51450edbffaad579b7dbf4b81bd979d" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954595-bEgJj2EaSBzq2ABMxxw8", "proof_sha256": "1e7e87bb791680d54c386cbb12d43fdc245d6f5f0cea12356bb11194761c84a2" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792954616-swz2cZaSgRG6BVI1MOoI", "proof_sha256": "f160eb6dd4a7911de119f0483bcb14d573c00ef2239f034522bf669fdd2fab61" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954656-QimTMUScluXMMP7RIiE4", "proof_sha256": "6fdd2a82916fd038f2d3ab15f32f97e65f3008c5c6f28283c4458e3c1376ab1c" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792954679-900egXZZJifHJTsYi74L", "proof_sha256": "70258fdeddd460f7452e468ecf39efbb7e2fee81e1cb99f47ed895301529af3a" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792954720-BoeLDxk33BKa4SGNxbw6", "proof_sha256": "3b1cda7b6c70386930de98651e608d469f825b6e58f60cfedd9ed48ea99bc14e" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792954761-T6df584aLRMWJqlSP4ZP", "proof_sha256": "e61a19c023cc9d19bbaf1d167fff40eea5d6a33a3bc77f40e572a8e3f1a2b44f" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792954801-hlf1sJHBYLAadSOP7BVO", "proof_sha256": "5debd1dd740f415297ae114fdb549b04220e85752ebae4ba39904c709effeb29" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954821-ZeqUb3ymtwL37KyzbAuJ", "proof_sha256": "5526679c8f7c45b6c71bf3d271f528767ec013115080f7a189536cc0881df496" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792954850-0qkkFKoiV8poIe25wU2d", "proof_sha256": "9c6d5f330c8b3c221b3dde47ab096ad86fd5d2b087b0396d1940bc75db035511" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "u = \\frac{\\tau{\\text{req}} - \\tau0}{c + (\\sigma{v0}+ \\", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954872-mBZz8CetIWKx2Hhu0KIW", "proof_sha256": "8cb79f9ae3d20e4157b1926aaf9c92ce4506da203c91f0e7e73b1e3f29478383" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f}}", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261025T200000Z", "generation_id": "gen-1792954912-hE6gtks2rcCupiU9l6m9", "proof_sha256": "e7c6687a9fdd6a08ac26cd387b4c0f6e244b36e4e5c7b8fedbac77f64a5267f9" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954930-z4SC2PA9zhABOCbe5ioR", "proof_sha256": "4812c1184e01ac20550466a16ed1cd6d27b2cbce15e56e9bc16276f585c67985" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261025T200000Z", "generation_id": "gen-1792954948-Q0Nwl2JySUMSIlgAAjDc", "proof_sha256": "bf49f41d59d4058ccb245ffdde9fb5ad0153f0cb21dcb4e2f5b0ac9ae15542f1" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792954961-I321CBNyTGfX1jJ1rVfO", "proof_sha256": "7b38af8292ac957d550679f285b5c0ca3d9598737d7a3dfed7f7a3f778acf193" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792954975-sdJfn2mNlOPTd6ohJTp4", "proof_sha256": "124a3608ca87b83688a43706ebfeaa3b38797ef12052085801e91cd00772915b" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792955016-ZiKfbhel3LTyyHeqcCEl", "proof_sha256": "87c59ddbcf4e61b758572611fd725045369a4e123bd080c8d257326ffce065af" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792955046-5JG3nV15UyO9bDt5HgFV", "proof_sha256": "c12143f80042a74f36c0c3ec6816daaf2bcba47b834e273c98df0e6686ed27db" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792955097-7dNqDfnl1XuGWyUF2HmD", "proof_sha256": "ab4edc3e620b10517815f226876ef2e2de3668040ff96db73c21ae810fce2ed0" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792955150-moBI9IApnrF759RQ0iKx", "proof_sha256": "711b8fd8caa50109bf367dfa09f9797a1391ec6f3ef7e309935cd63ef049a824" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792955226-BvWkW1VzME7P8J6xpUxe", "proof_sha256": "98514cd775d47199eb962844ebfc6505dcb6e2d2537cbef1e29d3a93052205b6" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792955301-DlYoIdTuRfnKXgHya4Pu", "proof_sha256": "68bdb58d5957620f5b8178b1a68158ff63fb70083f1f8a84de3aa79dd90422eb" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792955451-NeOG0uf5oHiI9a7wjty8", "proof_sha256": "d885393129dc30add2aa25d045d61b3a082c69fdff572f1ca113f7201db7deab" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792955506-5GEWqGxcQfNXG7KbhjxV", "proof_sha256": "b946b5f73a7d1bb4a62d1364794b78690f77ccc8e52ba93b709aaf3ac22d423f" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792955559-SRYEtvdRV97lMz4vNLMJ", "proof_sha256": "97b3da512c00a50ef644b80edae79a1ce43583c95dc2f6aad7e07098e1cb1ee3" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792955621-qpdDMmxZlNaTt4dvXtc7", "proof_sha256": "e1d454e19d8b68acbc59eb6a3d3bb95c8fdc9e6248217f8b81f91b71271cc3c1" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792955645-VJU37PyF1PsvqbyX7rBx", "proof_sha256": "5abf33ede33da85ebd4b05680855b924e91c3a33f0fab3b9adb781bd5df69e11" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955710-qw4cNs0CE0N9mr0eQxY6", "proof_sha256": "3a6a552fdc51e2f2063be8c7dc4c04f14c0395e80475f0d2cb3d3a5785e424c3" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792955797-aCH81yMoGIykoz6ezvSt", "proof_sha256": "655cb42e5f2ab7363707f3ba2cde51995e086598278640e0097f78da2885a2fd" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792955859-IeBpxfzKCPvlX5k7XGFM", "proof_sha256": "89ad903518137ac5e0a35202a4839c411a1a1faa32eae4bce799d5f9faa4e48c" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792955923-cobMrxvudLP0b7nDUX8L", "proof_sha256": "6b5d31dbbc3ebf04a15dbffe81b2ebc66a37fb8c2354678011b9804d476c68e1" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792955975-2gC4AIICXaKskqZwMjyd", "proof_sha256": "5e16007ed853d74f6a20358ba380b44cf858f8b49abd7d5a71ba70f8d51cb075" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792955998-XuTqi7Xei642iIbHjE9s", "proof_sha256": "5a326611d576617a87e3d99ef09a389dad49ea66c3fa2085d83f47a3649a1b01" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792956009-LkmeZzWKgAfUvFhhBQl8", "proof_sha256": "3f86e4857db863c0e0a3247c4825abcd205b9a989a4079effa22e29f9881b491" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792956019-TfRMX9fpL8gVoTdHMhWg", "proof_sha256": "a87b744b24bd0b2fd316bc21a7dd73345093665bcb319b3cf541bafa80c96d44" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792956045-0hlM8OjFxTIHNHzOVgfo", "proof_sha256": "37c39700a7a9375ae2bd7d4c2b658aa6a7d98f33fc6d84e501336f86c17fdb97" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792956057-N4vvxarNS3W9TRtYR9Hi", "proof_sha256": "92747cf049d90ac335dfa8d147581288a854967d8ce67d6774b609189a2eb41d" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792956067-koTGS0eCcvDSJu37zqSF", "proof_sha256": "9049ec9542c0a3dbc5780489ea50a7961aec27ef27242b59fe37d4aa25e687eb" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792956085-VrAKoF2XSvpNjwzjfm5N", "proof_sha256": "4a7d3b2de787a7ace36545fc7d0751fd68cf657d3fd63b4c14a43645bc34d9cb" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792956096-GZ8EdqcsVJJCDLVkd6eG", "proof_sha256": "b13a1542550d455d1804bdafb32442a450e4eed37a037d5cc027b819b2c19553" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792956111-rclruwjW4tr3bM0XkRF0", "proof_sha256": "bf4e6bdc95982701e1a4276a698d34d998cbfa68030afdead04a06625ef8340f" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792956121-HJ6tosl18S3aG31BAlQj", "proof_sha256": "b614c6ce80bac77915daeebda4902f7f120b13eb6fc27cc4c1e975686fdf89c8" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792956130-9cVJY0NhQjSU4SHBoRlR", "proof_sha256": "c7ba9e37e52baf81557c73eda4d203cf40d569f83bc387fcb13bdd9f5b5eabd8" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792956143-6XCMewv0Bse1SEcBDzMC", "proof_sha256": "dcd3c77efe28a22d5b124e0387f9cb2d85ee198c2b14798eab76c43073dc9e25" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792956155-vx6BUporDRN4zOf4AL2I", "proof_sha256": "d622e02e081f3269e2c9dafa48b8672d091b2f85665b5b13741c6ad9d923f54c" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792956165-esZPkzLPBAlsVHUqirXK", "proof_sha256": "8c08e156067af54fc701b33a55e4022e2e4b3338a85a8e8d6f9cdcd7ec4b1452" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792956173-RvCcEvPJHLYjmADJqnhs", "proof_sha256": "c36aa06510027bf9e7984e3e2d1776ecbaae10ba143fa4639dba90d031f713f0" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792956203-StkArPJuAnbmaHx62D9a", "proof_sha256": "146ede9df7557e5622c02085207bfa79ac6b395361190e4bf88730de5f6ab888" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792956213-ahSRrDYET8iXM6MRuoRL", "proof_sha256": "9c0ebd20ed2092a09c0c219c619300ca4bbe9011b9e88e8392235b0134732022" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792956233-N3efXBmMIsHF6hiBpYQS", "proof_sha256": "05e283b73d6082b886e47bd1275215dfecc0e1442d16ce219e205b2814df58a8" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "s = 2\\sum{k=1}^\\infty \\bigl(2^{-(k-1)^2", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792956241-nAhrOuKx71Cf93guivCu", "proof_sha256": "59770cd413eae97078cf1ae6b68b4599dd00edb197dd7856a23d1c5f231f25e1" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792956273-zwa0ESQS2ua6XAq2NmTt", "proof_sha256": "293956dc751124c28cebb12426f7853692462519111676eb21121a81eb0ceeb8" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792956282-C8DTMSfcsE2jtp7Pai0f", "proof_sha256": "f038e3f03e0bf4cec37252b54ff5ddcc0d2abd724e17ab3c45f7b4a496ec8366" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792956308-elgMzeGYRLLDj44MxSSH", "proof_sha256": "4ccee453cb6770f8991ac7148f2503334c2ea3f321526120a47d895f7e855274" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792956320-4AZy2bG80cr0VyT1WyDO", "proof_sha256": "d453d3d96f4774a3f2af8625e81955b15c6595edad465942df17ae3984f7cbf5" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792956339-3q8dlSWDAu9gJqZrNNjc", "proof_sha256": "f3bced6bf408f7649055f061700e974adad06135cb9317efd136a5d06a0b7ca6" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792956349-YdLGNZvIVJ4rJaqwM6bT", "proof_sha256": "a8833fd493ce2c6a904d43ce4d4f1c321f620c2f74fae73e770f9fada2d51a55" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T200000Z", "generation_id": "gen-1792956362-ecgG6yJo5FQuxbnP0t77", "proof_sha256": "9318a7d1648eb6ce0435833c26ea95f870ae7b363333cd0fbca2c23ddf2465c0" }, { "model": "nvidia/nemotron-3-ultra-550b-a55b", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "BaseTen", "run_id": "20261025T200000Z", "generation_id": "gen-1792956371-fB6yC1KSYb6JWoFAOFoh", "proof_sha256": "1325b744bcf19e4fbbaf6a512456ed6db497544a93cc62296e6f331ae40a7547" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792950837-7lAPfWK2R4ThEsFccppK", "proof_sha256": "7a0ae6530e3a1a064127152ac66d61bd978139d15fb914fc6d8cf603c44ed615" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792950859-LIEoHpGtu5KsgZ0mdqzZ", "proof_sha256": "108acade49b320c432a26a2dbf0f1d982ba0f1318a73273fc81d2f713d6c808c" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792950883-cZ28SfjGEIpoESeCwUO0", "proof_sha256": "6766f3c519f01ca2c26c2c37132fbcedaaba00d920948f6b67cca5c7f8aa1963" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "$$ \\text{numerator} = \\cos^3(x) + \\sin^2(x)\\cos(x) - \\sin^3(x) - \\sin(x)\\cos^2(x) - \\left( \\cos^3(x) - \\sin^3(x) - \\sin(x)\\cos^2(", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792954184-HTM2zqfsxqTWPfOXdu37", "proof_sha256": "12d36c69900d9c67f6b824eed72ebf69db52243b4573ad95f146dd72a812c45f" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792954256-wbBnRVMon5QMHvbQ9Xyf", "proof_sha256": "383d140f8651a8db408e37e5af7ac87e2982c2a49c0ad6f042f60d0b3f5b4664" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792954288-Az9gcnGHwoYStL9xG9EB", "proof_sha256": "58ca04dcc9c51616d068b1895f06990b84e107c623e55655999431af9688e1f0" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792954328-YKBcPlIpEtBBeGTcpgx7", "proof_sha256": "e8b1eb798e311642628b13e4fb1dca5ca0760ce47bb24b83ac2bc4ce2b3efb3f" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792954347-Kk8L4D6I8VpzKYW6U6PU", "proof_sha256": "7c2a80a73ae9e6cfb401586f29d591ee904f3ee67c77bf64a137b248ac6d6532" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792954379-SfqAJUAxdwLn3iaX7LOm", "proof_sha256": "01a69e8cfd80dc35c398ea226ec1c99b1b24b121b493e6cbf408d82a4db3f167" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792954401-c7Ljf88wwvvtVrxQgSPQ", "proof_sha256": "7b79ec0df32569df915563b8d8c47e4b6ab9ee063c2ef23e11372aa702a28ff8" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792954427-BacBFGF1ApeRUui24KcA", "proof_sha256": "4327c84f074677202f14481242eeab843ae4b4e9957868b6e3c8939951d62f0f" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "- $cd / cr =", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792954445-odKrGtihTDI5a4JCH2ef", "proof_sha256": "199b256c76a8d0381f539f11371f6ea35bd4abf52bd2ae01468cb07ecc35e1a4" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792954490-IIH4RvuPG5v2qmsnrvsq", "proof_sha256": "119c090fbcb159b7435c19285a128f4b848d64db2daf36d8a8d76ee48b56dfb0" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792954505-S5AWgSh7tzwLEDi3PsF0", "proof_sha256": "c0ccbd43197739e8480f2d121912223f11631126a3bec8f5183e59aae4e7cc67" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792954528-PNNcM16UnCEGqAs5A1iS", "proof_sha256": "7b79d3736dd1238a32d7259591014010af92b915a3a002953c93bbf2704b1bde" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792954553-8IhVSJV9Fj7eOIh5Z263", "proof_sha256": "093022f07268139386540218f528a8eafdaffaa4a7cbdeed4321ffffdeaeb0b0" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792954572-S9OgpMUAval3ECVfVful", "proof_sha256": "2eab0f3bfeb593449cbaf8fa1efd18fd23018e16edf7543a4ce82a6ab8433bd5" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792954592-PH05hyfM7o5JYBJ9zU3K", "proof_sha256": "ec7347778089beba4c3c6099d8efaf040b0adace0bd67c95709cd7b470d7baaa" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792954616-6VCYm80ckhhjL7oFpzHp", "proof_sha256": "55c2555eb7a5f6e01d3fced5439a7080f25d1a9cf1a25e105ac950ef7148d741" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792954639-br0iH9OSY4cnnLu7SRFz", "proof_sha256": "6bff5ff036fa461e4b975dd86074927726326a365969f60e81d06068765806f9" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792954657-zeWCCEWxuBvQRYuh25zn", "proof_sha256": "f4bdd949b13274e019fd9ac5df699dea66d2190378179f720c15de6749af4420" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792954674-BKNf1Le7c47AbykHrTCl", "proof_sha256": "10f2804e9f0a67d992faa524ca0a76ee69ec41a0f5c7f097a4e214aeb50428f0" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792954700-KWihv2LmyZRy7GKwhJIu", "proof_sha256": "fe60a4af1fe68ac37fe51006b85bd09119965e2275d0cd24bd1dd3b0115096b1" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792954727-XlcNnHXUZkJg3wgdRN2E", "proof_sha256": "164b576dfb0e644ae8cffb9757d22fe52dfb972e0f5340d1ea82bf594bc77f60" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792954750-cUpntwNLtAXMbbypgiiD", "proof_sha256": "661210d59d0327de68d1ee6b5820b6ff47d44a90cb9e5fa3b0327c5f44104d0f" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792954786-GlVv72C24XEszl69VKsb", "proof_sha256": "a8e4fd8f3320532c5d7ab8fd865ec2ced0dd6cf744a7ede7afb6bccc7dc0fdb8" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792954808-ldwDG00IxCu9TaqIRaSI", "proof_sha256": "39929c179a2d4dca7768a3ef5ce0932709489e009dc4b14053018df1d9a5cd7b" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792954841-7MEgXu2BnOgsMOGvQZV5", "proof_sha256": "49ccdbb09119b66ca4a30875fed62d3f08d013955f9f71863be5c97a486876f9" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792954876-EEm3L5tfQJWewbWa16xN", "proof_sha256": "b4f7a3f5f27c4f902905a4b970220ae53a3bc3de2821897123c9f6a86e979997" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792954911-jlXJ9RyWeLdtU3wUgC6L", "proof_sha256": "4ab565d4fad2fdd6cba016c2140aa87a4ad72b2d00e5187f8454d1bed2a943e1" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "$$ \\tau = \\frac{0.06}{2} \\times \\frac{999.1 \\times 9.807 \\times 2.3}{100} \\approx 6.7607 \\, \\mathrm{n/m^", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792954934-fyZUKu6FCcXkHCdTfyW8", "proof_sha256": "9547f97da6e127747a08f7b4fce520950fdbc19fadeead4ca4eaa0dab7c8a3d3" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792954980-Y8QteXfyk1nG834ksRKw", "proof_sha256": "3551a82bd1b9e7a72810a873c4418c45f9410e568f15e85c7c8671efc417576a" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792955006-qaacPpsxNJlusOI2J9IX", "proof_sha256": "f12cd080221edde4abcf990750e870b8aac2d8fe233c41180cee7f793f2c85ae" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "- $", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792955032-ogVaePDlZTarzkYAfY2J", "proof_sha256": "680ca7fda6bf035c733afb544d4593b487fa28220958e9b18ef6766c27a3bd94" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792955077-xUdPtu6UKDLQuQlD8avX", "proof_sha256": "6606ff555a1c5a9d6e2fdf7f601fdd5f53245c83131cd12e6caf1c5d535f5aad" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792955097-zjpN0o0XVI7dN6URv13Z", "proof_sha256": "5426585ae604f8e7b3ab48eac3eacc0c9a5f1aff6d859581ba023212cda9a105" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792955123-lYR76JvbZ2xcFLQY6ug8", "proof_sha256": "a51cba12d570e9c76c0450e0d07613c9258e955d9ea4991515b82efd3c0288cc" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792955143-HjwEhtSdtI5Y7sBZ5pNl", "proof_sha256": "bb93fd619915b3dbf5abe241ac965df0e8fba0f396a9d32e014dc31dc41c18f5" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792955171-0CZuKCJrQy1ub4jl7d84", "proof_sha256": "985f8113da5d8f296a081eca996138076d63a6934970a546ac45c6495aac22f7" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792955186-J2WWW2pAk8fZwpcHA4x8", "proof_sha256": "c56558ea898954d5a4353fee5bf8e11948f9ff23d5157d89fe0989c5d3d8ffdd" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792955200-9GyMG583teahpNZIO15e", "proof_sha256": "6a35fccb463458eb2fab9ad7ed0ef349cb1e3c1bc6c10f356bfb86ae255a9725" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792955214-tCUjtUQ87wjxc3Xy3yLI", "proof_sha256": "87e51f82c60aa6fbee24b98c26ef20286f3307c1fa0ee7b3e42849b599c79579" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792955229-MLKG206ejz0IWdglTFtv", "proof_sha256": "d9d09b3786756aabe29c3a73446ddf0c0dbf829861f541e21e59774afbfd14c5" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792955248-Bcv75ap1ZatLMnrBLrau", "proof_sha256": "9b014a0e4b18d300210a83d60f47dffa6a082d8d7550df42dce851f9533d08b3" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792955291-Nmuha4i6f1zBdoaJCEsu", "proof_sha256": "59db4780409a0cd8e59a36b25f77d2472ad337ca2a213a07c2a404761ab3f97d" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792955309-6rkqs5j8EGmuxJkVLzP1", "proof_sha256": "e3c0c6c5de7b8d4118944b072a03a76829686e7ee6672f4ce1475ce176b68140" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792955356-jEFuVdQYTk63XIg1R2hh", "proof_sha256": "194820a11744d585d9bce18414172de0b622866e157e2aa75bbc4738ec0c1bc8" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792955403-rPmExOtCSHSszH31LAOB", "proof_sha256": "4620677d9bef5684ba184521aaa66edb5aac27350490696e4e77ca2bdbef7359" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792955465-nRNVf3SlO8mp4vyQTRnX", "proof_sha256": "cc7dddb6a3c8d6c6dea3adb03795aa6f775eee824a87de84b939e1b3c4250d1f" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792955514-wbtGeM1aM32rvUiSLhqM", "proof_sha256": "9fcae8927b6e60caf22e804342b7ee85b474fd859efaf992de2cb89d331bb165" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792955558-eNCqpBhgCWuLRjlWKmvp", "proof_sha256": "d3ff6c51f89059a105a4ec50bfce7efdbff68525958781fa6cbabb3ed4bd9756" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792955613-nFq4zdPCuphsupzyrXzi", "proof_sha256": "d4fa571af8408a3114dd72143b75a02ac6fff58a5384175ca9dee058b4be4793" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792955637-lRvoLHoU9NBcXxNZBebs", "proof_sha256": "1156c161ad7dfc2ec65490d363c952035602c8fd398377b6db8480c3e32252ed" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792955653-tUW1EwL4Jm8B1sKOHfEE", "proof_sha256": "ab7478b7faeb8b480934ce0bd1f68c751ca125532d40e642012c10e5e5d7a207" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792955717-fg7xAVehkpygBw9WbZHh", "proof_sha256": "01f7bd754fdc363497ea47871e91b124bb51dbda3bf95b958f34a2516291c28a" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792955765-Z1KzywAYvDwFmmaXoQPh", "proof_sha256": "38001f4d84411ed1e1a4b45524ca3379316da7374a251db6c539da443540dee9" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792955797-ftEJgbtjzAgIE1VHoJ3W", "proof_sha256": "30e678f19509bfe4e9a89c0a46c9a9da401fcb23c57a0d123f889ee2b261d6e3" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792955842-bS6wHanqdoQOMnImdBHW", "proof_sha256": "5a1df4b05afa5edce2ea58007b79ce14bcabd954e5ec82c41496ea3a70c933b5" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792955885-2jRu1MOCgS5i8DaWHo2l", "proof_sha256": "7f00e93388d25e8bdfc618ae092ad3c5fc6f9aa227b7463444095ac28f463578" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792955929-n5Vw1jXMb3uFNka3Bq8k", "proof_sha256": "8a139fe0b35b1e843c7821fe1862b15bd6dfe7d492f50a274eecb78bb12985b6" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792955971-luiN4zOr2mlPSs6pQ6a7", "proof_sha256": "16d67d5eafe98336ffd208ba7912accd5d1e6c4220ba2ed46340b47b20933977" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792955993-NFocXqjrcquT60C5gGAL", "proof_sha256": "e93e888666b4ce0d7560079f6b4bf0f17f02c72ffd3841d5bd00d6744ca0048b" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792956014-voYicVWq4WSLxgWLsYQa", "proof_sha256": "01d1aeb3d384f1a89f4741b64e068e46c266e66507b6261017b361e8657a0db9" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792956026-Fz0WvXxRObWL37IIGBDG", "proof_sha256": "26d019d1baa9828c09ffb28c9c1c71decd0972a28bc9223e84ef6c9113ee8e46" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792956041-il49gZNHin6dGeULWHOh", "proof_sha256": "2fa4b00ba2262228ba6e7a0cb825776403caa4bf1565da381b893d42b283ed0c" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792956055-0g9ZfejE3ULD8sQWbQhm", "proof_sha256": "59ece7a51c53e146e8cfc87e147a4acc265d33b61c9fd3392ba5031c64e4a179" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792956067-Uf2GiqO8Ys3947wT6Ts1", "proof_sha256": "f91b891acec721bc250179ea381af9c2bcd3286b4a1222e85f4e0d9c63551a83" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792956080-EmaeXSaC2OypRlzC8DDZ", "proof_sha256": "3301d5ff7ea88b5c43b112f62addbb366bf7e5c43b2893919ba665912e95dc65" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792956094-XrzMHZQ2EVz8WcEGhvhb", "proof_sha256": "00f13284d8df55af60961c9c32d36c4d03da097f44c0aab03c09c24db88885ab" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792956105-fwn8BzcJ167qJDTnDXvb", "proof_sha256": "cc93da0a1f3e13501ce4d0f26c0ffa86728dc4e4ecb181696bdc61be761736df" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792956118-peGDbtSV73NVJcCkjho0", "proof_sha256": "903d78018a381436868a5d373a0dd7bf1f31fcb048f7063f1ea9c88e35ee3e66" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792956130-f1TpStjeGXxiHWPrOxLz", "proof_sha256": "c5b688a746162059fe6f60183a8f2b6b904fdad05c2c04df5138dc74d8a5cad4" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792956157-kbfpNNIEpbBZ3OJyjt1k", "proof_sha256": "1c8b5b6f8c85878b46165011beb15fa18caf52dcb72a1d44885c1fd19fc95659" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792956171-2lARoPn4kk5bqoEvYdpY", "proof_sha256": "0e40ba4946dff48bda7e84e4c20d0e30a7257f50357c7dab3a41880c88b16e2e" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792956203-X5euVKpVfih3zb3MMS7m", "proof_sha256": "4ce6f4bcc4603d11129915a2a94afca7abb05dc81b9a374a722da75187c3f9b6" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792956219-9Ho9ye9IzPzksktOHHRR", "proof_sha256": "ce5cd4622264b45a231955e4f3ea5f92748b8ec9946df0c822a0db2081a4414b" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792956232-JXiC7YriMjhH9aVO449Y", "proof_sha256": "337c15d08c4cbe725bd506b0cf3df5954df963b4d36600a12fd2e5ec564bde10" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792956255-zOVFjXTqkWwzgy1zNfRl", "proof_sha256": "d61e2bfec9b6a44f804afa88a39f9a2a8fde4c74973b69e83d0dbcedc0453f5c" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792956269-1AhomLpmkj7OH9hTK5ae", "proof_sha256": "af0c981e22c6a0b5b899b001ca7c2b5025a88f489690a99f69fe3b2a1ab110a7" }, { "model": "google/gemini-3.1-pro-preview", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T195000Z", "generation_id": "gen-1792956280-KQkxHtg2k182BsE7AW0r", "proof_sha256": "4538fe4acfaa248ab26bbc5ceaa301180376fe13c618328a4f82588a3952a09e" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T195000Z", "generation_id": "gen-1792950884-4OIT0AEKkGKC9OAV9fNW", "proof_sha256": "114b968d4714f9a81114219a185299b47fdd0bfe9439b94aa9f5b99b4f8f1616" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261025T195000Z", "generation_id": "gen-1792950895-9lxmm5UTxN6g6j3tc5RS", "proof_sha256": "04feff363332acf312f30ce57ecf168fbbff4da560b2474f307efea75db37bb1" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T195000Z", "generation_id": "gen-1792954184-Sk79DQFfLr8whBuJI0cf", "proof_sha256": "e7badd4d7d98f328b0448b8618b89c889d372ea20803658a53e596ae9b1cfbcf" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Baidu", "run_id": "20261025T195000Z", "generation_id": "gen-1792954256-GxOVB42dfcPFFjEoQK3s", "proof_sha256": "f3db295fd590535dc6d836894f3681331c698afd0b9a237d799f9691174d2f42" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "GMICloud", "run_id": "20261025T195000Z", "generation_id": "gen-1792954291-3oSTlsXtG3bAoICNcc99", "proof_sha256": "b947773f262d9f1e277a1b988299fc848f218b46ce8d6aecd8d01c63a6aa0d2c" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Ionstream", "run_id": "20261025T195000Z", "generation_id": "gen-1792954330-yWqlDHm2PVcfWtAgs5Iz", "proof_sha256": "9377298db8b409757825222de2c0c6590f49cbf83bee9d28b80ad212874e173a" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261025T195000Z", "generation_id": "gen-1792954354-3oSqdc9ukitzzwWBvnDK", "proof_sha256": "7977d95ce23a2a6bc39136dfb932caa274afe3842c9b28867151d7cb441c2402" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261025T195000Z", "generation_id": "gen-1792954363-43cZLLjInNh8lZzP6WXj", "proof_sha256": "56325327930b0059c66f79a18c65d866289eff3418132c73b1c181640888d350" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261025T195000Z", "generation_id": "gen-1792954384-B4ZwLJ5HAGBCuFFebFYZ", "proof_sha256": "6cebfab41cdd0f1b10aa3d9c14ca40a9c0aa4cb5700edbb05610fd724f5fd5ff" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Ionstream", "run_id": "20261025T195000Z", "generation_id": "gen-1792954397-ZK8ghZyvlYxIafljZ3Z3", "proof_sha256": "2a41f3d43c88243ad882afcbf68c9202d9fc2c221234dc7ec3bca603ac1b9d6a" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T195000Z", "generation_id": "gen-1792954456-eU7JssvOl7ggx2Ncq0Kn", "proof_sha256": "5aa0738cd96d0ccda8a7cd39d54bb51f55d2b3cb51eef126e2d85b394827b3b1" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Sail Research", "run_id": "20261025T195000Z", "generation_id": "gen-1792954464-1eBec7WUms854RjIh7i7", "proof_sha256": "4b687fac84e43fb164f70626f9230f00a6d14451777b4bd722fd4d3d89ac9208" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261025T195000Z", "generation_id": "gen-1792954569-ZKDopxpq45KOLf6bwvHJ", "proof_sha256": "919c197e0219128979911a844b001071659e8afa6db200d21c33e70736e0f6a4" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Sail Research", "run_id": "20261025T195000Z", "generation_id": "gen-1792954577-Q9jsAgQg6NrGtAE6SlAL", "proof_sha256": "fda3085e7af2a7aa2a4291a1ea58bd59ec28decaaa1a398e1c17f233b65b30c9" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Baidu", "run_id": "20261025T195000Z", "generation_id": "gen-1792954596-Ejui4ruFL3IqtsDHmqi4", "proof_sha256": "afb1491932cd8de531daa5859c4518ee5574e5ba40a4f35ecebadb2a74fce7c2" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T195000Z", "generation_id": "gen-1792954644-fuoQ6WeV1mW37LqiqdC6", "proof_sha256": "2f1efeb62be2e190d84764cbb147feb357b23cfd3fe6cefae60165c6577d8c3b" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T195000Z", "generation_id": "gen-1792954653-kUqg9izUL7r9lo8GV0kK", "proof_sha256": "64459e4277630dfe33b23a596b00f07915829f71eb4c4eebd57b4d14e41774dd" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T195000Z", "generation_id": "gen-1792954663-NPSXG8FopIJeaq7JBIb6", "proof_sha256": "e56fb3c2168d4ef014871da25bf76fa2c6d2aa752f00046b2055989dc03dbf0f" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T195000Z", "generation_id": "gen-1792954675-fZK411sl0L0nWMI8o2QX", "proof_sha256": "b11a45e34a5cbdf10e6169f4ddb1f06ea6f358ae224377d4f0f76cf36aaf8dfc" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T195000Z", "generation_id": "gen-1792954692-Jp5PmM1mUYBaehhUjvdr", "proof_sha256": "fdff1ab85e6459106f49977290194b8d3e5b9e00fbf58ccf8ffe67584ca858c1" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Baidu", "run_id": "20261025T195000Z", "generation_id": "gen-1792954701-bCOYErwMofZqQIsIcl1p", "proof_sha256": "c876595f3d2e2bb20162352c9ed6e2ceb82d94f584858a1a72d3f03490295f88" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T195000Z", "generation_id": "gen-1792954714-mGKgpfB00VH38brF8966", "proof_sha256": "0f5453029a6188d1692d10dda9cc4540e40f2938be3cef6586d2625ff6f1c29d" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Wafer", "run_id": "20261025T195000Z", "generation_id": "gen-1792954726-eXSpxWIISXnnefyW2Thb", "proof_sha256": "da549851d686c2ecb3dfde95c3fe273692c42fc9bf0dbb476ebbac77525c3c8f" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Baidu", "run_id": "20261025T195000Z", "generation_id": "gen-1792954752-Vd6A9tZtoi5pgHc5MSNL", "proof_sha256": "69a280fda5b81071f343b89a0b3cef637c6661e92bd4c601ad80b3cb414a107d" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261025T195000Z", "generation_id": "gen-1792954773-SFdgoAp343qttbBIC80T", "proof_sha256": "574c42fe4d295eaef8eb6c4ffd6f2e90b964e9cae69e437a00765e042b55c2d0" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Together", "run_id": "20261025T195000Z", "generation_id": "gen-1792954800-LUt9Pp88RBaQ4PMMkAUx", "proof_sha256": "ebab0346e1dc37144467570a0710067601f54bbe4769b2fe6055ce99e464e886" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T195000Z", "generation_id": "gen-1792954832-kMRawLN8TrMadE3MjnVr", "proof_sha256": "2157ec7f9411844033ec36e8f2ac56bd10b0c1ba75fb09bd0a12271fb92e0bf3" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T195000Z", "generation_id": "gen-1792954846-5sUOaEcauiBEoOkToRs0", "proof_sha256": "dff7d332f0128f4f01111867b2e436074163242303571ecd26f7b1bad9d0cfc6" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Alibaba", "run_id": "20261025T195000Z", "generation_id": "gen-1792954862-AaL7NzdShPt5hWZtBUOg", "proof_sha256": "09452a67ca25f440c5f0ea69276b923f4dab55aed4f79af36cea5c5c51874b22" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "CoreWeave", "run_id": "20261025T195000Z", "generation_id": "gen-1792954944-RCo8cqNGzYFWcuSlaegl", "proof_sha256": "3339de796c33c0bb70590583488554c3471d6352a506abc5f7bf4e3ca82c09a9" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Baidu", "run_id": "20261025T195000Z", "generation_id": "gen-1792954975-JLiYyPYUWz9kkEm2ZfNL", "proof_sha256": "ad5a51d4012e8e500ed7202393cf897117d4eaa76acaf4491d911178e34eedc3" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T195000Z", "generation_id": "gen-1792955040-uL4XoDMTJOAs8EBVKMtq", "proof_sha256": "5a6e6f7d4e253dfe0ea97a970aab264305e829f04de44a7d57f5a079196d3605" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T195000Z", "generation_id": "gen-1792955057-CHRMYHGG7CtyaWQoQRZM", "proof_sha256": "881ea2dd6f626f0762c995256259cff911ddbe4c731629f3fd57f8c24b753834" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Baidu", "run_id": "20261025T195000Z", "generation_id": "gen-1792955068-HFNMPdK6Er5LK1jFs4Fb", "proof_sha256": "ee9c4e5720d0af959fe0eef63b3e1522809e24b0f09c477dc994e1ed2323b258" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261025T195000Z", "generation_id": "gen-1792955138-G37EvKNSmUvWeo4L5PYM", "proof_sha256": "164e78fba08a22fbc579fa7ac3f7b0a3c5170c42969e4b5aa6c22b2703f616f0" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T195000Z", "generation_id": "gen-1792955150-IH2yeao99OBL6A0LRObT", "proof_sha256": "9f08b544586551c5f71879f3bc626540b9a5c765311a78f7d6ab4c1dd229f5a5" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T195000Z", "generation_id": "gen-1792955171-4bkp5XUacEFHtW0Zy6gq", "proof_sha256": "61be2e1ea335a67bfec1df31501436d9100461ff1711f4dbeaf2ffd08a730af6" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Sail Research", "run_id": "20261025T195000Z", "generation_id": "gen-1792955180-ZsJ2KHDuKUYZ6wYimWcP", "proof_sha256": "7ee09514cbfca846dfc77be7b953ce5bc1f419a4ec7b7b26c76ef770acc32bdc" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T195000Z", "generation_id": "gen-1792955316-J6o9K955h63qVJcbazPS", "proof_sha256": "cdc910a01d8f47e36ebbaf86ab774f982547b677f82300c72f5f60d8ff54dca0" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T195000Z", "generation_id": "gen-1792955335-jYjLoVSGlZ28P74AJIUJ", "proof_sha256": "3c42752103ea5768f96f4a0ca00d6dc2b3ddd29b277005d4523ed1f4ef72aaa0" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T195000Z", "generation_id": "gen-1792955382-8rswWECK9Lz7PXUpgXFu", "proof_sha256": "8a0cfb10ecaf4c8becf5087027786880be400e48806a9c16faf2e120d6cdbbf5" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T195000Z", "generation_id": "gen-1792955437-osoWqPTPFBRQt5BZmtTK", "proof_sha256": "bfb9f82dd88cf51693b3cc34058acbc32f9033b5e8a0441fe482984b2eed606c" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Baidu", "run_id": "20261025T195000Z", "generation_id": "gen-1792955499-m0CRDbJuASwgLtQMt2I3", "proof_sha256": "bb18971e2fc8ab7aad2ae4c5aa03bd45c24cd279cd02c80417b1b1d2a84fe401" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T195000Z", "generation_id": "gen-1792955546-O24jG0MfBZBKijdkiLwr", "proof_sha256": "2dfad365eb509f90596e5d4c030b80dff12389c2daff386f9b31e0dbfa387940" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "StreamLake", "run_id": "20261025T195000Z", "generation_id": "gen-1792955601-4FyzXWqPb9hhU9egGmdd", "proof_sha256": "f34ba5165085024b5bb654543c0817885750427d882561fb2a1e1a26994fdeff" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T195000Z", "generation_id": "gen-1792955636-5Y0oCpMisbWKN8ZQLoci", "proof_sha256": "f0ee7d53c290fa3eaed3529b7a0079883453fb15df7c1ab338cca96a5070a228" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Sail Research", "run_id": "20261025T195000Z", "generation_id": "gen-1792955644-h4B1rGWe0w9HGJPkS9CW", "proof_sha256": "4610f2b0ba76ffd29953478445ac1cd4685042c4f76468001baa646389a7f062" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Alibaba", "run_id": "20261025T195000Z", "generation_id": "gen-1792955655-eICu98ZBjBG1oBXIfeWy", "proof_sha256": "ecad2f6c70657f4495becc40b0283463d69a496567def322b0e2997c72e16170" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Baidu", "run_id": "20261025T195000Z", "generation_id": "gen-1792955745-HZqWpyLWeYVBnsWTTPvT", "proof_sha256": "30e306369e2d8bddb50a9f6b6e993a1ac1cd8a8ec156a2a3be0d24111eb461d6" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T195000Z", "generation_id": "gen-1792955797-lCNmKWWadBP8mdZpnGSz", "proof_sha256": "d8006a7df1212c6812c293603a21cb4576ceb0e2297bdfa789892135a16ac587" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Baidu", "run_id": "20261025T195000Z", "generation_id": "gen-1792955844-bGiVDENESaGiqVCdrNhM", "proof_sha256": "7dfa9a939a75007bf90991d3d492896c2adc8a3a76b94270233cf31a76067c7f" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T195000Z", "generation_id": "gen-1792955957-D1UctHi0UYDtwrb4iNsO", "proof_sha256": "42d58ec71f92f02e8492283e971414c136fd0bb93887f0f86d4ff94a5edd7296" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Phala", "run_id": "20261025T195000Z", "generation_id": "gen-1792955995-9Taiuxmq6CVjLcMkg22p", "proof_sha256": "2a970577f8a80eeee26aad07533ea0aa6fadae1f852f6a7e59fdf5952d0c52f4" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Baidu", "run_id": "20261025T195000Z", "generation_id": "gen-1792956018-iyf9mb0QpKXW7jswgvLs", "proof_sha256": "762b070978aa60672ee79ef8488fd82c31b6402a20b115179c50a3e5c44dd331" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Baidu", "run_id": "20261025T195000Z", "generation_id": "gen-1792956032-EMrNkcs9fOwGcZnXCuWb", "proof_sha256": "583ad92f7fb4b601ddfed73f64511ee89ebf290d57c6a353adea8a6dc99bada8" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Alibaba", "run_id": "20261025T195000Z", "generation_id": "gen-1792956108-qDSgwdX9E8P9HS0VQASx", "proof_sha256": "8ec4d79fc577a55ad5be88af23cf76a0f6ffeca7813776539a0dd58fd9d58d84" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Baidu", "run_id": "20261025T195000Z", "generation_id": "gen-1792956178-hurDnvdIj6891WR860w0", "proof_sha256": "a9400e988daa65decc8e4d5cb3be6e3e4eb73738dfa7b649752d8ddf16b377d7" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T195000Z", "generation_id": "gen-1792956190-SX35J8Z4lqL6b45Jm9hV", "proof_sha256": "0680719f8caf21a87f00b5f6a629591693b4c121c1b7e000579b41785edef2ad" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Baidu", "run_id": "20261025T195000Z", "generation_id": "gen-1792956198-i07Hpu0WKMeHw8FYPnFk", "proof_sha256": "eacfd099107e9d771be13d832fdf815732aedfd95a4ffeac9cc43204e7a7053a" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T195000Z", "generation_id": "gen-1792956209-97wDfZp81MKIRMgHwLmO", "proof_sha256": "92bdb7882c6c0006489577dda3614d6b4e9096a22ae05ef450038b8c215664da" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T195000Z", "generation_id": "gen-1792956218-Gvk0ksecAukd0X5FIbgF", "proof_sha256": "f0dca3397b3149b62cf1ba50baad1e01cbbe5aa3830db2382ec06f25393567a3" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T195000Z", "generation_id": "gen-1792956235-nU36Ylj3G2ihcJddShR6", "proof_sha256": "fb0abf8237abfea19f8ed8d501076b6f8710a63bc6b32bdd1b6711267c7efc5a" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DekaLLM", "run_id": "20261025T195000Z", "generation_id": "gen-1792956253-B10rpqdHNvqN4pFH3LAk", "proof_sha256": "d0a6a89a460f0c067978a4417baffc8d69943ac455d02ca2a32534ebed3163c5" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T195000Z", "generation_id": "gen-1792956262-oS3KLhhSwXVovh0mIzGg", "proof_sha256": "4a53d0eac1f533ef68c0a70776210baafe3b30741ce86837ddf943acb5a6c54b" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Baidu", "run_id": "20261025T195000Z", "generation_id": "gen-1792956271-PvAAmJhrMqHyIC93Letv", "proof_sha256": "3be954c91c4782dc09ef08ff0e110c9428f7deba28c5569443cb9f6c3aaf2a3f" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Sail Research", "run_id": "20261025T195000Z", "generation_id": "gen-1792956282-fbxU3Q5ojxHLUVth9VsA", "proof_sha256": "7da8cb686bb8d87bf549ee4f015e0b4adb8eda3ff136715015f42a42dfe2e288" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "StreamLake", "run_id": "20261025T195000Z", "generation_id": "gen-1792956294-WELrIwbbbeHXBZCD0WPJ", "proof_sha256": "561223b63bc37924367088ffa1274674aca0128b3e1612200e5d2e650f2f3717" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "StreamLake", "run_id": "20261025T195000Z", "generation_id": "gen-1792956316-xoSdjstZKpMloQyi6eTn", "proof_sha256": "225790b76b79a5e3a335285e328037f03979ca0e92031d5a27bd5f0710d8febf" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Baidu", "run_id": "20261025T195000Z", "generation_id": "gen-1792956332-K9btK1izzmNg6upHF5U1", "proof_sha256": "114d40f74ccd59f649f1e266ebd16f839800f111644b437b1603d61075799366" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T195000Z", "generation_id": "gen-1792956352-Lnxli56PUQSjaDJ2fN5V", "proof_sha256": "dff1f165797b333b06f6551333e08e0257f7f5f8be001d71358fcc88d39f519f" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Baidu", "run_id": "20261025T195000Z", "generation_id": "gen-1792956361-fyEh8WIW1gPHcGeZsigS", "proof_sha256": "d73f0217fb5e0faef549c0edfd1e09693d7bd6fe24b88857bcb202a405d09f60" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Ionstream", "run_id": "20261025T195000Z", "generation_id": "gen-1792956370-U7uXQdpIhBjAiySi2xne", "proof_sha256": "ee6f03c33dc017ff098456b8d45a2ac58cf7c9c29d1e60e5f58b74e53105a68f" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T195000Z", "generation_id": "gen-1792956391-WGptUSxb4ADnIQmOgkYY", "proof_sha256": "f8a1a55ec65bb142d3fb6db6148888bc5032e27c4430651f936d335c533b317c" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T195000Z", "generation_id": "gen-1792956400-xsk0IxqyiVpFPKBOiZcA", "proof_sha256": "472f49f17ee1b368d901f8e3429f2a897a8446c01d9320530857c0b91f09bd38" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T195000Z", "generation_id": "gen-1792956412-wc5AoQvmHG57kQi9Fbuj", "proof_sha256": "0c9d7f4d29b20d81dc9481c7e5518cfec279cd463f78e4267a6229cdfad0cec2" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Baidu", "run_id": "20261025T195000Z", "generation_id": "gen-1792956424-P3ykHnb5YgXw355WQehJ", "proof_sha256": "d351a5f76e0bc5f1245cfe185da8077ff28a35113a22000e2ec4692f3fba9e13" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T195000Z", "generation_id": "gen-1792956436-9fw7ZJObDL4fcumx6M1c", "proof_sha256": "a8b4ab99dd20136a19d54b44a3e4f8b2726bd96e7ff18ecc0a6d6f50b51e1071" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "StreamLake", "run_id": "20261025T195000Z", "generation_id": "gen-1792956450-zwTZi8aK1B2TBExrD8qX", "proof_sha256": "457d2586dd8e767a5650794e3a28b4bfd05444e852835317fb166fb490126633" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T195000Z", "generation_id": "gen-1792956460-q8AfWVRBZSCiEu5dVAlM", "proof_sha256": "95070b9d77712ea2e5d06603603d4b35e7272d9ecf65b086faf191e22fe5e7dc" }, { "model": "deepseek/deepseek-v4-pro-0813", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T195000Z", "generation_id": "gen-1792956467-ThHwdt0G5HREqX8EFhSc", "proof_sha256": "1aa7b8d1852e07620bc0edb84c346995cb79453ae3be56d2151db1197d810ac9" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792954184-2Ha5cYoaGYf0kAhySwS0", "proof_sha256": "9ff8b481e1452187f85b73fe2f589f25227148e72bef4484bca19f7b08ef559e" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792954256-f7wyPCkQa1ncHjEMTF6h", "proof_sha256": "2827ae20e8a5c84ccdab5a416134b371942dff80ea9eefe59cb66e48bedef0cd" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792954291-H85tLGarIODvAhZxTVuh", "proof_sha256": "6ed1f418bbfc26dfb6eda0670831673d11dc26ee3e1cf37b4c053a0cd525d491" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792954316-zcPO4kvSsZlLJoLMxjOn", "proof_sha256": "108ab90d9a5fd97c2a3c5bc857dfb39b240c70c8db6bf25842b1751c0c823342" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792954347-EVXsIUHWEcjvdxItaooy", "proof_sha256": "8394be8342d13a2616932323cbcc553570fc9b055c0d2328c88095a40a50f03f" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792954383-UhENhVVdH6xxDg9nW4OT", "proof_sha256": "214357581e344b77d80ef65e3ba890255bd19614600f40ba92e458c715f025e7" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792954432-1wPkhgBsJoXhLUFVcWX1", "proof_sha256": "78d22cf40d7b425b52714c8be3808cd582351ae91774f431e7e82d4acd4ba653" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792954445-Fjn88Ihw86eksli8Rby1", "proof_sha256": "adde0516ff55cd679d9867bdde04c6a0c5348f3537b639add35e612d571daa57" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792954518-BErlx8cogojl5KgyYYAZ", "proof_sha256": "00bccdb91ab7a9b51328627df7e75e7c011e9fdacecf83d43c70ed464e4b9080" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792954561-EiAco70nbVO6G6aqtoEP", "proof_sha256": "8fb3d018663458ee9079a9f38fe2fe67e16d617107a789ec2fe635b0b8cdc8ba" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792954595-Ui9G5TWsbcOUEqR99eb6", "proof_sha256": "cc622aae8f55e50c604cc764b5614765ba2fe2e44ada5c1e402c971929ae016b" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792954606-HiGPCM1rUFBG08sVF5lz", "proof_sha256": "92b01377106c87fd971d6d307cf38819012360721b8cfff1c37b717f5a89dcc9" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792954637-6a14vzgX0I5LFbLfMzzV", "proof_sha256": "6c5afa9faa61c40df294be927e970fb311e57ead9c6439bae6b508303722ccda" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792954649-IdmKLnm9Hyhkp3yM82MA", "proof_sha256": "9341a105b00da224ec963f09589def2e6d18ac0492025f5ac4a57a78e0f576e3" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792954667-uLoATyd5zmOOZSkB3GRs", "proof_sha256": "0ecaf103a33cd3394b0d3601b47bd0c31afe6796d93abda8be792f1faf0439d7" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792954703-KDTsIizrqfCt8WRnGR0C", "proof_sha256": "ba99ed0a557ef9227e275c0e5072f9d1443eba4115ed00a1700321bbed4fe5f8" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792954739-d8VhDJJGMlRPoflNwvhc", "proof_sha256": "16e5caad5ec69e81b1237c5dbc7c386ad87d0aa5fdca35d99d259e47e319f8da" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792954790-ri1e0ygFIo5K64iEHswZ", "proof_sha256": "02513ad2c04eab8d298f322446f74821016fe9ffe2cd87d3bfd6f843bf5321ce" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792954810-HcnLl9oWCY3jHdOUfWn9", "proof_sha256": "058a35cca800343cbde6b51b77b2f57a93e03638d2a968dbf4c90b82a42d3f36" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792954829-C0pYTL0UJeqWQgZpPZoE", "proof_sha256": "173d12ce0029a64b4975d63cd54aae6ef24d67e3a2a1deaaeee6334e865e7d1a" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792954841-BVGhs74wNZZhzBKdXevu", "proof_sha256": "b3c86afca9738b21082c640b148ee0ed16e670be282232cc749c0be494899a89" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792954854-xg9rfICK4NbxtTC9P2UK", "proof_sha256": "fac9a97bd2d7fa854c1eae464d425239786899918c20981634659732a0f45388" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792954899-rwwNvOvObXkhXt5fPd3h", "proof_sha256": "29774e790f38c06bde2fb419ea3d11445b6dcd5f569ebbae7a1cbb21d2550b93" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792954951-j5anuA1aKORJnhuNrHRc", "proof_sha256": "23b7e706c56d8cb1baf81845fa0fa238dc5221595cbe9eed71750951a46395a0" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792954982-G9SrXCpt6nqiLRQSRDAW", "proof_sha256": "3a2407eaaa74abf5e9c6fb39273a2a2a647b98ab7a6c7c0e7e1aaf78c21061e6" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792955019-KUmhkwMY1l9elqO8UEmb", "proof_sha256": "cc07b50f2bfbf59d7d3ca2dd526cde35bb0a37c03f4c2979a215f176ab2e3ad0" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792955064-MZnYJsc3sZuKqNeYWudS", "proof_sha256": "f901fd8dfa29cec752a86b8c523d25ff910def2c009f3a99961e953cd69b4d08" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792955097-VhsTxivlddsJvtBFgwQg", "proof_sha256": "0848be025ef307c36d55bf1cff91b1bd245398662a6d400dcd9bd92844fcf332" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792955133-ReoPLiYOaQKeYGbQUdVg", "proof_sha256": "edcceabb98dbca95893530d3e59a636327d5128ff3ad3ea1575c6aa44aa48056" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792955188-COn9RuHhYjqZitwB6V0i", "proof_sha256": "06dfc7d0dd4db3bda5e051d6b61e988a1f04668742094131f31acb1b5adb97cc" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792955214-4DXEemv7cvJ1ivdyoZ6O", "proof_sha256": "6b5a38f2cb0913f184db890a9f1d223d6312128f3bdc4441065a8715c2f25d4f" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792955261-PNYyl0Ivmrsgq4YaWt8c", "proof_sha256": "e1b1dcaf62981d1a06e3a3af7b019b142c1648f8e2e741b2d6c0f543eaf4326e" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792955306-dMTMZwaqUR73BbVC2y2N", "proof_sha256": "ead3a8fff728df47cc957af0197a4c770aa3cfa74238763b38fc82c618d50f4f" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792955346-5iRQop92j8reDuRzGtPC", "proof_sha256": "1a10344f99fc27502f885a1977b2bfe4f6bb71a1259cf47879b26005671200e8" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792955405-cN2Fu456S0HHFyDSsAcE", "proof_sha256": "885ecb19eb69cdeea17aefc6eddc492ce32fc7910329d3bc1c1e6ca0783db006" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792955474-qUdAbjPROt0IxTPUSuEy", "proof_sha256": "ae9289fe768b25e07f6fc1ffef2b45d4813f342a8ca0126f75ff6f449e1cda9c" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792955538-4AGczrXK98ZMUsSfH7RM", "proof_sha256": "507c87883b91f046685ca047d935b3aef5e20485a2a69f60c6e552a3d8c1b0d1" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792955605-Zm5qWyBAq6i6ChPmYoXq", "proof_sha256": "5d1c15a57801ac7d8435079952f6c1e23827ebf49691bba3440e9658c10085c4" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792955652-qBkjg98fOODY9M4wEhaz", "proof_sha256": "5e45038e01952a8ddaf5678740ca1a8ab0735b10d05ec6aadc473f146736a593" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792955708-mCMqcaAdp2AD0dUNHrqV", "proof_sha256": "a83b5c5e8366fc271854007b1376c4aba9efe54d537815ef2de9022025eb0a17" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792955797-DFZmfu8WTINK9siVCvKn", "proof_sha256": "0c12b67354d08cc6b38348d4a3fc369207d354ac464faf60cbe32d2f27e6898e" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792955889-7sDUVzcMXgnFkVV1y7wI", "proof_sha256": "4b05df042c4a8f1b6e42e72a11e22eebef7cb8038126c4259aa2469ac3fc48a2" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792955964-ZBUUzmOqbQfNBVViNxPK", "proof_sha256": "18a273ef45b025eed0b600eda3e579bf0e7c3e06da71f192a30e88a4e26ccd38" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792956010-FxP4ExZtjKI6yZubB8A4", "proof_sha256": "54d633417b97e0acb3c4678dd9419eafb9d44319163abccaa1bf429aab76a261" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792956046-PzG2vxtkXPk9A8CodvlS", "proof_sha256": "9a6d4a33061f9ff7ad1c6481f322678e608e52296f9d2f7f3e2798a40983c1df" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792956060-5aqs6a9MerpTn2cl6uNE", "proof_sha256": "bc54e4f12fdf6cc1480fe9ed7f7bf5d180ff4098d3c5db441c8851786ae1628a" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792956095-sreosmyVO0KoYEc29viB", "proof_sha256": "b2cbae5287977db434e6d9a249140dfa1262f38d2958b4e37139fafbea6efbe2" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792956107-z6MMU1oFEaR7klibFl3v", "proof_sha256": "1fdaf347b549948c47ac4e25f48cc75b506f26f574ec165af39c7036335fe33c" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792956140-CRMRjsVmTZtTqrIHbX5p", "proof_sha256": "149fa1144cf65f20156ba130f64d4ff398b6566150b178931e7fd3e6737de3ba" }, { "model": "mistralai/mistral-large-4-0", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792956190-OjjOybrGWUsKPayq5hvK", "proof_sha256": "a27cf6818f50957d7992095b45586625c3ca9e75dacac6222e9a4f306098f5ca" }, { "model": "mistralai/mistral-large-4-0", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792956202-x20k1joPxCzGEHMNw3Re", "proof_sha256": "a2a6d6d1e0b390b19eae33cf583afef56fceca5d750a221694794155ac70b2f1" }, { "model": "mistralai/mistral-large-4-0", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792956242-7wHVDxm2f9bZwt5dmXJL", "proof_sha256": "f42d39632099d945d24ad3a97242b42f68ceaf025f1e115908cbb3b9ced648d7" }, { "model": "mistralai/mistral-large-4-0", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792956274-2GyHgqpl36CRoUR5BYi2", "proof_sha256": "73a1a8242e9bf91798a1c192f1e28d7012931e1c1a229686618b22ea0cbed936" }, { "model": "mistralai/mistral-large-4-0", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792956303-mhfZkOqahybXh3BoHzC7", "proof_sha256": "14603c73699330c4c0c89fc009a0a012fd0655108fbc3741b1a110446b329b2e" }, { "model": "mistralai/mistral-large-4-0", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792956367-lo3KJVYiDhB9cTgFlWPm", "proof_sha256": "cdc84deae87b7d7558bf087af8f6dbc0520212bd59b910eb028514b8629c45d9" }, { "model": "mistralai/mistral-large-4-0", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792956435-SWFsxqkdJqNkEAtGJ893", "proof_sha256": "69992814840165119ddf298c8ac72778d042fe9ebf1ad93106b3854366ba136b" }, { "model": "mistralai/mistral-large-4-0", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792956471-Svp76OFuQCMVwOaGu8ei", "proof_sha256": "e5de932009506424ec95cb171e80dd319fd902a39cd6c5a794bcd708a2f5deac" }, { "model": "mistralai/mistral-large-4-0", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792956503-byECcrYtzZhWkBHHs8JN", "proof_sha256": "c035b5e51501ee032adf9dec9d84daf1d67a5cd502035ea7085e8d752dde933a" }, { "model": "mistralai/mistral-large-4-0", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792956512-Jkk5JeAli2r72VSpaXRu", "proof_sha256": "cc3fd67a49ed70a92b1eaad57b35a23b1a02c192c4a4ddd67922cde7a3ed9a61" }, { "model": "mistralai/mistral-large-4-0", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792956527-hE6P6NjG3eXxJPanFWDM", "proof_sha256": "b7b7da586486ff20d6e088734ebdf2d8a1a42b9959787d5bec4318fa5e186f6b" }, { "model": "mistralai/mistral-large-4-0", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792956562-thnplYr5DiEuoMjKa5bO", "proof_sha256": "14d9f0803a4db1abf61cb3a15093ce2c614e0d9e19ef14548a57e8f637443840" }, { "model": "mistralai/mistral-large-4-0", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792956579-8qfw3o57oSQfe8ExYfQS", "proof_sha256": "dcaba8c3eb7a5cfe0df31f77bb438a672d42dff799d82faaa4e3daedbfdbd7bb" }, { "model": "mistralai/mistral-large-4-0", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792956600-I8GdOhMSWl7tDQHCiW1O", "proof_sha256": "4cf639150492756859668884161f0f3fd3fcd3bd0b014a775f6f2c96671c9cbc" }, { "model": "mistralai/mistral-large-4-0", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792956617-V7IWyNdZnCKnGzkGeNXM", "proof_sha256": "3904107c3a8110bd75a348a52803ae7eafd89a0bd6095b7c4ecffd9f6190bcf7" }, { "model": "mistralai/mistral-large-4-0", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792956631-zZlZQa99Ci9ljkulO2V0", "proof_sha256": "e06e6e784a031b8b2e91a63604169b0048ea6c1e8ddcfc2e3e0bdfcaf3774774" }, { "model": "mistralai/mistral-large-4-0", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792956641-8f0Dmv7Q0n9khknSpcun", "proof_sha256": "9d11be14fa3bd65ea9e656b401ecfbd7ff46c6c34ee7f774b989a6fec50a0eac" }, { "model": "mistralai/mistral-large-4-0", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792956659-5CAilDfxwmsvsLCpzUw3", "proof_sha256": "d698e5684a9dcdd7066515a2d89ffcf27efe82124e7e52dac8d04df7738bf5f0" }, { "model": "mistralai/mistral-large-4-0", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792956669-CE8vohwKQAvFd1c6pnUn", "proof_sha256": "728966ff6e1061541b9ee95c2c5a1a2c5cd4588899b35214b50e2992fc42b62a" }, { "model": "mistralai/mistral-large-4-0", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792956702-BbXxMau37kyurJccWgN4", "proof_sha256": "8bb06b201cc43693dbdcb43817180a76e000c11bf338f9a7648d90e4fd2230be" }, { "model": "mistralai/mistral-large-4-0", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792956710-307IsfBXs31NTboTk5EZ", "proof_sha256": "d63f15df53c60190a55ac16933a7d31a266d415a0cdc21aba941c66c96d2d12b" }, { "model": "mistralai/mistral-large-4-0", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792956719-cn028UO89Y75MpFwfJbc", "proof_sha256": "e7af274a44aad231433cee5e8752211fced4dffbd98b6778a1814a0e63ede615" }, { "model": "mistralai/mistral-large-4-0", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792956729-SivJo05jNVAPGtQLREHr", "proof_sha256": "67835c9b0070c027aee466b525d0ebcbf08bb05c57df819426aba33ab2e2f879" }, { "model": "mistralai/mistral-large-4-0", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792956753-Nu02qRTLVWV2Nk6lEpMf", "proof_sha256": "2d7359133ce716838ba1cd97b96721a8372b12fd851fb7875f16bc566f5006b2" }, { "model": "mistralai/mistral-large-4-0", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792956762-2tgdpAL8n3TRUDGGTd22", "proof_sha256": "687fc002c6e7d6ba8390468f568d5f5855446ab626f8fc7f89f3fbcaf844f5ea" }, { "model": "mistralai/mistral-large-4-0", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792956792-9WAc113ZvUT97rYLBUHP", "proof_sha256": "bb2692747cf9af7069f60a0883ebdb89ee34d5bd109fb6ff8077baa9dc7e8eee" }, { "model": "mistralai/mistral-large-4-0", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792956810-ImQHuO6EV6pVtVGmf82B", "proof_sha256": "6c792d1f6c342ec8d0704c1568581d1526d33c165aea531724080d1b797e57e1" }, { "model": "mistralai/mistral-large-4-0", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792956825-WAXYGiwUjYXUytGOWzfH", "proof_sha256": "5d826a6db4dcb84d054778d6df6e824b056c798578351f5774a6166586de187e" }, { "model": "mistralai/mistral-large-4-0", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792956849-FxQhiR7yKzGkdCMclFPX", "proof_sha256": "dc66392bc3745b4726a008c525bfeb25f7e5f435c33577e69d2f2b50ef0a5540" }, { "model": "mistralai/mistral-large-4-0", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792956861-B6enMPuWccQJsYpRUdCV", "proof_sha256": "c2ce7dfd393af1ba17f955e3f2da7d3e71c6731c7acccb9ee081ce62f9b791fe" }, { "model": "mistralai/mistral-large-4-0", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mistral", "run_id": "20261025T195000Z", "generation_id": "gen-1792956869-ESCZFJBkbqgaMvc0edWm", "proof_sha256": "0dca93e0955f8cfaeff4db2ed8682084b946418b5f6aae8f0c271397d6e3a63e" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261025T195000Z", "generation_id": "gen-1792954184-idHuLRJ8lbEgNtVGq8VQ", "proof_sha256": "c8fe585427f111eef9da301c5c0d768ce639ee369c17e654ce6b6d1b3a8daa59" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "e - qua c - both c and e are correct; c is normally quoted as div. for |x|>3", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261025T195000Z", "generation_id": "gen-1792954199-RpuwPaGSwHapNvXm7Pbo", "proof_sha256": "7d94ef3efdc76d568b39725daffbd44c4d93af458b99e2465cbdb7eb51f37ae6" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Makora", "run_id": "20261025T195000Z", "generation_id": "gen-1792954217-L4nvL1DofLlRozzXDRDW", "proof_sha256": "ceb77d4ed7d0c3fc7c219877d8b0be748fc39ccf55733c69f2d46e1c4e732694" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "BaseTen", "run_id": "20261025T195000Z", "generation_id": "gen-1792954256-TWMLbfnsAky7NWocvbo2", "proof_sha256": "076d80b4bac429b774eda346addef881eee2393b2509c3178cbf9018ad42039b" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261025T195000Z", "generation_id": "gen-1792954269-k57T3YxKhs55vJw0R2GQ", "proof_sha256": "b03081935f9e77dd3f5628a6815055eb370c8ac95d979beb61bf54c759836a10" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Fireworks", "run_id": "20261025T195000Z", "generation_id": "gen-1792954307-kT0YPx9m1nFEEwEtSVp7", "proof_sha256": "61c7490b6a886bd3431573a88d188da129515b148d5c7425923bf099815b2931" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261025T195000Z", "generation_id": "gen-1792954325-YH7GLotHCoJzXPkvRcUG", "proof_sha256": "8c2099cba608fc9ec8f3232fb00c5e7a0cddd04899ff41daf5e00992febcd768" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Makora", "run_id": "20261025T195000Z", "generation_id": "gen-1792954342-xpi04JeepQx6llSkXA5T", "proof_sha256": "1866927a2550de435aea276c1bb63a913409754169f311eb0632491152db0a2c" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Makora", "run_id": "20261025T195000Z", "generation_id": "gen-1792954351-HI9n4PWCLwMBJPGfX7pX", "proof_sha256": "8d5507d1a0383978308bd61f58e41349e52e151aa71976d8326c46ffbeffad34" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "c - ${\\cal s}=\\sqrt{3}\\approx 1.7\\approx 2$ - ${\\cal s}^{2}=3$ is b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "BaseTen", "run_id": "20261025T195000Z", "generation_id": "gen-1792954360-AA8fdBdWLTy4WjKLjylC", "proof_sha256": "df633fba13b18b741d85cf7499db0f1f581c5923a7f1ff2df908afa4ef00e769" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261025T195000Z", "generation_id": "gen-1792954373-hIHvaFqE6FprIeaZvxe7", "proof_sha256": "a33d224cbfbfad74786efcb8ba1bd710444897f761f3ab3d4cb71e34fcede55e" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Fireworks", "run_id": "20261025T195000Z", "generation_id": "gen-1792954382-SPrmadQWJLeKMcXGua3A", "proof_sha256": "948a0b91b7545acff014f3d6c3a2515e4a02f4a11c415045cb015dec3dd127c5" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Modal", "run_id": "20261025T195000Z", "generation_id": "gen-1792954433-PWqPy5ud97RFwXoTkeVt", "proof_sha256": "bddb1e89cb3a91a9e05a5a537969a560909e4e2c4488615569e89834e8b2571c" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261025T195000Z", "generation_id": "gen-1792954442-rfdJNusUDsBWrbxNDUaH", "proof_sha256": "dc66196e4c1d831d62ad1b0738d8982046620d492a262a0326b8265572fcb31b" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Fireworks", "run_id": "20261025T195000Z", "generation_id": "gen-1792954452-IRnoHP8TTB55bWuSajzc", "proof_sha256": "2e73332be136e69372b8acea2d959721e67b926e4816eadb313a4058a74a8c31" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "BaseTen", "run_id": "20261025T195000Z", "generation_id": "gen-1792954472-3GzFONQNm3A2SOBZ7s2m", "proof_sha256": "d153809efb862f6de88f8b028a73410e1b55ba5bf9b8ac4ceffe35d5889022b8" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Makora", "run_id": "20261025T195000Z", "generation_id": "gen-1792954483-olilwdiBWMcEBRsrYruT", "proof_sha256": "e28579cb9213b2f802c1f46ee0492652efea21f2068fd96ba392630f3eb56fb5" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261025T195000Z", "generation_id": "gen-1792954492-pw4Jmtf7vCHl9AnWiIRW", "proof_sha256": "894b830d3a88be97bdcc9f032c609c87420cb0256e299ca7e7032b28edea4450" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Fireworks", "run_id": "20261025T195000Z", "generation_id": "gen-1792954506-rawsSZvt4XJnxgkKWrjc", "proof_sha256": "d38bfbcb3899958b8111fa9b4d80bbf7e0b7fb8aa3733d1f6ecfde99b74badf4" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Fireworks", "run_id": "20261025T195000Z", "generation_id": "gen-1792954521-GcFQgTpN3XmX0WC1ULGP", "proof_sha256": "4904323241c5f6015b18d812438d762386be4fae434e6ce4572a304a33081e83" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261025T195000Z", "generation_id": "gen-1792954532-ZkhJzhytQvcWj3yDWWkp", "proof_sha256": "527aabfce45cafbf9ae94aea010990e31d5b05412323f953c748953b1b26beaf" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261025T195000Z", "generation_id": "gen-1792954550-yn4bG6F6JpyMllYBKKrO", "proof_sha256": "cf09ae3130c0b5f87544be2a4bc9e2841f436af7ffced5c9e6275e8e49de368c" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261025T195000Z", "generation_id": "gen-1792954561-0nqyUeAxEpKED4CNDrHg", "proof_sha256": "d019b233536067fc913e31024feb72e792cfed439595d8a41b515e6f6839febb" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261025T195000Z", "generation_id": "gen-1792954576-vwuHoBi6KqroD9HYMZoi", "proof_sha256": "dce66dc93b4e053e220f3c1f0233a367710d35e1ba1095e189b28431658a791f" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "j for the ${\\rm air}$ thermal thickness, ${\\rm \\approx 7.5\\,mm}$, ${\\rm f\\approx 7.0\\,mm}$ for the velocity thickness; for ${\\rm h{2}o}$ ${\\rm \\delta \\approx 1.2{\\rm -}1.3\\,mm\\approx e}$", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261025T195000Z", "generation_id": "gen-1792954585-hhF08vkZkjxD5RN32er4", "proof_sha256": "803c64bf8a72a797a1df9684c62ee14d3de0d0fed6833e48704026c4ed1bc025" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Fireworks", "run_id": "20261025T195000Z", "generation_id": "gen-1792954615-1X21uZPlUAJdO5RFPVNe", "proof_sha256": "af593fc019d8d9d5661e2716c39dd9ab1f73bbd8885f814b31bb35684b595b07" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261025T195000Z", "generation_id": "gen-1792954645-vmhMdbeVtL16dRHxfBNX", "proof_sha256": "4f159a8fd41be2e6d8bfbc727e6e94286773a7227d68ca578c1df817aac8f123" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "e - ca. $18\\%$ for the conventional brewster", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261025T195000Z", "generation_id": "gen-1792954662-Qt11KaYE1gXaFQJuNBce", "proof_sha256": "1feeb88603b0669b1bff3853634a6f7e16ce15ec091c20a1576fff8ba9ab26c1" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Modal", "run_id": "20261025T195000Z", "generation_id": "gen-1792954679-IUlJcC31HkGWlKvjeCKb", "proof_sha256": "16e80d1f0a84e027e7171ebebe266ae750a246856c64db1d411e3bc2cbb9f219" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261025T195000Z", "generation_id": "gen-1792954691-PCdZVfmDaN4lds9jIHid", "proof_sha256": "c0e01c3eb9d62d7bb2c3a6ae5da674cf470773b3aec0fbc419d58b5d779fbe44" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "BaseTen", "run_id": "20261025T195000Z", "generation_id": "gen-1792954712-5ylAPZGKmRTXovb7pW9I", "proof_sha256": "7d113489e84a473f4b0e411590f762f58a49d0e5048aea78b367d3a478622385" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Makora", "run_id": "20261025T195000Z", "generation_id": "gen-1792954723-2cbLFTxOkOzNO89I9QEP", "proof_sha256": "b1f1502fd9b876cbc7263e948af8471e1bf87c1182d214e26933d93b44425848" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261025T195000Z", "generation_id": "gen-1792954735-xbLGvVu8z9is4HypgXOA", "proof_sha256": "776bc9402f8cc125fafd87ed5f328b931f97881643ddc4af32b4b930a1c1b069" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261025T195000Z", "generation_id": "gen-1792954753-Gu121btv45jrlfWDjSEm", "proof_sha256": "4d7fbc934a186448329bc5db79e52ed8bd3db1b49968205d762222845887ca7f" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Modal", "run_id": "20261025T195000Z", "generation_id": "gen-1792954764-C7fhR9mIQ9EqqawWRAp5", "proof_sha256": "4b9c8b6b4ecd4fcd1d5eab0413ceba370ceb36495043b697b1188a7baea44c11" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261025T195000Z", "generation_id": "gen-1792954773-gOwMYvMxtsRI6y5FxXEI", "proof_sha256": "ddc703776c792ecb32088f2a01dcdc39c3ae0153e350d64dd642ea1c1262a2a4" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "BaseTen", "run_id": "20261025T195000Z", "generation_id": "gen-1792954782-jQLygfV3bRnKHx1mhRH1", "proof_sha256": "d4bd7f6d7d888df1a5e8ae6af1c8fa1471a19d19febb13215d64c20f667261a5" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261025T195000Z", "generation_id": "gen-1792954791-Zzv0yKCuq0s38AbcOI6C", "proof_sha256": "78935386f251f873c5d9c03d9081dce8cf3d02ef2a7d38539a0d54ec12c4e0fe" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261025T195000Z", "generation_id": "gen-1792954806-mOHt2x9TVn93MnLXiHO4", "proof_sha256": "d3273244a423f4228c3bc37cfeceaca5ceb8d2e99ce8e502885a236c192b2d49" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261025T195000Z", "generation_id": "gen-1792954818-1L3sYdvGNH3U9e0t2YYE", "proof_sha256": "cb44c7e993d9ef6322e5c654d68856d8030d32d9ea269903eaca0da7c556eea8" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Makora", "run_id": "20261025T195000Z", "generation_id": "gen-1792954827-uadfgEtQYjhUFNrEEe9D", "proof_sha256": "fa9745b159207c094d587a9b6483702acb2748c17dffaf9a21bf171ca0b56ede" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "j - j. acanthamoeba polyphaga = f. a.castellanii = i. a.culbertsoni = e. acanthamoeba spp. = b. acanthamoeba sensu lato", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Makora", "run_id": "20261025T195000Z", "generation_id": "gen-1792954836-SRgjEN2rnsImeh0uvz5m", "proof_sha256": "9437b41dc9ee552e769312d794e7aa644658c25dacca694ca710436d6b276ddc" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261025T195000Z", "generation_id": "gen-1792954847-k1FpzGtBeNChnxKuYZxK", "proof_sha256": "0579e1eda5d282193fe0b6e10e287941f0aaa7d234d34e02b7ae9ed12e11ada2" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "f with decay correction, a without, nominally", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261025T195000Z", "generation_id": "gen-1792954865-QhJ4gw2yn8z8sKlFcg64", "proof_sha256": "31f14b6ddad4c06c9d0ee76aef1c77ffe220784854d0a1629881db3957067dd6" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "BaseTen", "run_id": "20261025T195000Z", "generation_id": "gen-1792954911-unZCDR1mR2W0eo5eP05a", "proof_sha256": "97353faf9f09a0a5fa435cb9d6b87b64b67d865b91a9beaf07fce3f21ea946f0" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261025T195000Z", "generation_id": "gen-1792954919-TYSNYTXDLhJLVDSQ3vOk", "proof_sha256": "a9fb28c8de3b61f4fca634ea0b624309e4831363d69d4e51f9f758c921934ad2" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Makora", "run_id": "20261025T195000Z", "generation_id": "gen-1792954928-AENxM4dGb4CmJaggGWSU", "proof_sha256": "9f0d537640f14e36cacd898e4dce646caa02490dcde18994e4a757b0d4e84b71" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Modal", "run_id": "20261025T195000Z", "generation_id": "gen-1792954937-8xFuKr0NPExRWDjkZ5Uy", "proof_sha256": "d99d813ef194a2b13014c4ce19def755800ca4717fec6688aef9b35acf487f9e" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261025T195000Z", "generation_id": "gen-1792954949-WWjy8zfOjWaGGEcmFn3d", "proof_sha256": "86870eafca5caf7137f51f5fd0efc2452e5c2b46ee2716f9f4621b8165bcd094" }, { "model": "meta/muse-spark-open", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261025T195000Z", "generation_id": "gen-1792954971-3zEAPF1zKpUaNa0pSIEG", "proof_sha256": "73ae31dbef82dca2e730222e91415deefeefa7bf52de0cfaae44e84f78b83dfa" }, { "model": "meta/muse-spark-open", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Makora", "run_id": "20261025T195000Z", "generation_id": "gen-1792954984-S4Tx1hu1asrkGFoJrNaR", "proof_sha256": "da153b02c4a123895f005591058d78c31ffdb43a0fc9c29ee3216374b3c1fa08" }, { "model": "meta/muse-spark-open", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261025T195000Z", "generation_id": "gen-1792954994-cS5T6fpI7GU2BtRsCcvV", "proof_sha256": "a3f9958bc5455744129a40c6473fb7bfa3b01ff8e1a990aa17e55641ebdefdf0" }, { "model": "meta/muse-spark-open", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "BaseTen", "run_id": "20261025T195000Z", "generation_id": "gen-1792955004-VHN8y4kRiDFrcQSf5bvv", "proof_sha256": "c6cf5fff1045e1105831ec51fff3c434532086f2216c51de784fcfd36aee4b1c" }, { "model": "meta/muse-spark-open", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Makora", "run_id": "20261025T195000Z", "generation_id": "gen-1792955012-XmOvmcSR174tvkE6slzH", "proof_sha256": "9ef20b5e6aa398e04acccc6211251ba8655f57da313b68c684b472258471e120" }, { "model": "meta/muse-spark-open", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261025T195000Z", "generation_id": "gen-1792955021-Btu0oaLUzrQv1MP7JHqI", "proof_sha256": "b38c25225747f2d8f21c495c5f1280f022b922b29af47d6f21abf2a1893b1404" }, { "model": "meta/muse-spark-open", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261025T195000Z", "generation_id": "gen-1792955036-a3Mb2dlBOtMep72mDiP5", "proof_sha256": "46b10c3210cd8ab02c56f17567db0d30cfaa245c41b438fb94bfefd15237867a" }, { "model": "meta/muse-spark-open", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Fireworks", "run_id": "20261025T195000Z", "generation_id": "gen-1792955054-aP18Z52e07vGpQLjp9Ew", "proof_sha256": "fc0f516800e353357a65bcde0dedc4dcf8ff77a9e8fa8d3d193c019c7a1fa643" }, { "model": "meta/muse-spark-open", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Makora", "run_id": "20261025T195000Z", "generation_id": "gen-1792955063-qneNg909w8j77SUfjWaz", "proof_sha256": "d0575441ec10cfa5f1cd090a719e50147136d2bc30df2d48fdf97fe03089fa5b" }, { "model": "meta/muse-spark-open", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261025T195000Z", "generation_id": "gen-1792955071-jL91TshteIeJD2hdqB0N", "proof_sha256": "3bae24c11025d37ead75a56bb05fd5fc1edaa3b20e53e58a138985f95ed01f2d" }, { "model": "meta/muse-spark-open", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "BaseTen", "run_id": "20261025T195000Z", "generation_id": "gen-1792955080-O5RpGkuqzF5Xijak9uOc", "proof_sha256": "8b247c7e6dba7fdb1595a44a79bc81d114ef4c742dfdb362864950da404df6ab" }, { "model": "meta/muse-spark-open", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261025T195000Z", "generation_id": "gen-1792955098-0pvyYFH5pZ1pY7Js9oD1", "proof_sha256": "783cfb0b67e2f0044f9e8e0244339c4fc019e426271730befe358cfe560d435b" }, { "model": "meta/muse-spark-open", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Modal", "run_id": "20261025T195000Z", "generation_id": "gen-1792955107-Mmz0W53NjSd5EjAffgGD", "proof_sha256": "0059ef4092901fc413bed1c4d6bcb3c17c7e64f9ae6c28e9270fc9d12951d132" }, { "model": "meta/muse-spark-open", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Modal", "run_id": "20261025T195000Z", "generation_id": "gen-1792955124-PO31QQKiSbkHZz9q6EFR", "proof_sha256": "a4c133cd15be505d3ddc21e4c1070dfb8b2fd7554ddcb42e78dfeb21ae17a038" }, { "model": "meta/muse-spark-open", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Modal", "run_id": "20261025T195000Z", "generation_id": "gen-1792955132-mYC3K3B2S0yWLHCWRxCO", "proof_sha256": "38a3e4056782ff69ef44f86b670ae7137a375267f54e0d49c207d6e694f3bdfb" }, { "model": "meta/muse-spark-open", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261025T195000Z", "generation_id": "gen-1792955140-KHnKqO3E7jIf9gwuX4B8", "proof_sha256": "52cbe9a026598b238a47dbeb23837b9f0d7f9d1026e0372a5d6f41e31c9888bd" }, { "model": "meta/muse-spark-open", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Fireworks", "run_id": "20261025T195000Z", "generation_id": "gen-1792955150-ATFgzSdFAH3BscP6rXsG", "proof_sha256": "9cf48904964eb2b1619b791c78d7ac473566801bb1dad1e38a2e69356da2720e" }, { "model": "meta/muse-spark-open", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261025T195000Z", "generation_id": "gen-1792955159-qPd2jv7WhaXaGJG7WNyX", "proof_sha256": "fb64f11ee7c784cb64f6550aad06921cde51678549b9c5144fdb36af76f8d38f" }, { "model": "meta/muse-spark-open", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "BaseTen", "run_id": "20261025T195000Z", "generation_id": "gen-1792955168-0oW8aKNhJqOgQsvwB5kx", "proof_sha256": "b2ad7b60d71d2621ff324009c6eabe349e2928512ed8e3c086514da18cefdcf1" }, { "model": "meta/muse-spark-open", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Modal", "run_id": "20261025T195000Z", "generation_id": "gen-1792955176-qLcdJPDhog0yGTjMQKvU", "proof_sha256": "f95efd3770edf9624a86be6f70f9d12a4ed111fdfcda9f4d5b15b4737de4be50" }, { "model": "meta/muse-spark-open", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Modal", "run_id": "20261025T195000Z", "generation_id": "gen-1792955184-jSywmeuVsamGUyBI9HvM", "proof_sha256": "93365dd626321a87b66f1a18b36f3e68e58fa7ef43da729c5f1d60416f9e597f" }, { "model": "meta/muse-spark-open", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261025T195000Z", "generation_id": "gen-1792955191-k0ZgXeUq7oh5ic60Ag28", "proof_sha256": "077ee96c245945b3dc33f01973d52185ac348e9c2cd0e49087d3056ff63e1c5e" }, { "model": "meta/muse-spark-open", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "BaseTen", "run_id": "20261025T195000Z", "generation_id": "gen-1792955200-G1pcBqMW4MLpTJJgroEU", "proof_sha256": "87d0b25f20f9dd96e92d40f4ca6b2667745865eadf98c1ef38e3cdc1ec976497" }, { "model": "meta/muse-spark-open", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Modal", "run_id": "20261025T195000Z", "generation_id": "gen-1792955211-bBxmh0IdVJ0z9gqU8OlA", "proof_sha256": "26c2a9e34d05b959bad5ddfe5ff077999d54d050e1f5154b58846e096705270e" }, { "model": "meta/muse-spark-open", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261025T195000Z", "generation_id": "gen-1792955219-t3a2DWm6wDHXG4uOgmEk", "proof_sha256": "e9de19c0eb4b340cfc20986190a0f482db645fb8c1e27ec9b8e73913dc8aa789" }, { "model": "meta/muse-spark-open", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "BaseTen", "run_id": "20261025T195000Z", "generation_id": "gen-1792955229-O5Tn8YfCLC6Ffwflr1o1", "proof_sha256": "e78b3b564b5d3cbb72f02040022cee72b12cc71c1ae7109daba0ba7883776471" }, { "model": "meta/muse-spark-open", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Fireworks", "run_id": "20261025T195000Z", "generation_id": "gen-1792955238-6Uuwcucoz351hOcc48cc", "proof_sha256": "3b7b918078ac26a38b915003c1a60029535e5d42fcd390e50962354f3c3b22a7" }, { "model": "meta/muse-spark-open", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Fireworks", "run_id": "20261025T195000Z", "generation_id": "gen-1792955248-oKcVNkKqnzms3Ek5KTqf", "proof_sha256": "b9f3d9100235736203e15b46c20b5de7aceb8459b13e3e746527805d790ba21d" }, { "model": "meta/muse-spark-open", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "BaseTen", "run_id": "20261025T195000Z", "generation_id": "gen-1792955262-IhEYexZeKXiS8i5RXDnn", "proof_sha256": "9cc4fb11ab074c1ed2706863f9174e0e78590d8358682aea785d83b1fd60a9da" }, { "model": "meta/muse-spark-open", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261025T195000Z", "generation_id": "gen-1792955270-unjENK8UVhPUxIaD9NMt", "proof_sha256": "226d174a554516d2c6625c2d17308966fc32657794157d6638898da33d41d5c4" }, { "model": "meta/muse-spark-open", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261025T195000Z", "generation_id": "gen-1792955279-AXZQxKDLWqVeBkl5fyYX", "proof_sha256": "7e06f9294213d3992a95007368d9a92b8ecb749f02da256e34898df19ca433f1" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792949253-mO8CmWeBz02VuEHLdg13", "proof_sha256": "624483a0d22f2b2bcc635fcd7436862a3a4679a36b461807dc2be91e434eb616" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792949267-rBrpp3G2t217MrUCVsXf", "proof_sha256": "873f50751f6b19bfde4e9eadcb265435804c34439b5245ba8510c5de072ae363" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792949293-CgHeE2zYbrlI9tWJsjjX", "proof_sha256": "bd676e400051a8d679eb39131bb2bcbf644361b918b6c5cb5ff64522b4023bad" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792949314-ZA8tAINfcsDzUMHhQp67", "proof_sha256": "8a7b80bc243a99a0ef18acd382262c206ceff7efce666dea40645c49dbae6966" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792949344-5gFQBzzhjDHNrlFVCOtR", "proof_sha256": "40c8984ec95ea2acac5afb6f30a24fcfebfcd357b98349fdfca6fcd0a56f7df5" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792949361-a2xmjL91zbd4KQNzpWqx", "proof_sha256": "7f0f29d6664f0e86e3bfaa39547acf59fae5c5c757a39f9d8e1772c72d0292ca" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792949377-27L647KcdjWX14Mtl34h", "proof_sha256": "0778367211efe9fb474617b7ed77605b3fc25b30b537e346b8466eab8a2bb4c3" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792949389-vSaORYG3AC6YyYesJOdU", "proof_sha256": "13b11d821354298e643af0d89e3d3c958795be6a1fd884ad1362bb055c96ceb8" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792949407-oYtidvgXuhtiXoMSEYoY", "proof_sha256": "259ea43ff330223e9c049530379808ea055236760da94f1e0831c61d463a4bba" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792949425-RauYt4alr7VNOXLQ8gMF", "proof_sha256": "b282d6e789c8fbb6b32dc71bfdd854fe196616de56ce92efebd954d55f4662bc" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792949450-OmI525GICBn4RBrjMtyd", "proof_sha256": "a86a39997be6b0aba65f12b2e4da103e3e82f4ec5dd1b8b0d448750b16c3b709" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792949484-US1nhfMxzZQbWM2SwDIy", "proof_sha256": "1f7022b60f42b1ad98053e89619281be3290a2349e2add2a29351ae48a3c4994" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792949513-5hC6BTVsvwpkJ4kvf4SW", "proof_sha256": "e0921eaf2c249422f0c754098a23ba18a251ed2b0f5dd2e1b9e28b9be70ab788" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792949532-jLxqAyR3Ng7O8L8YoHT7", "proof_sha256": "6e68279ea88ca36a64952db0354a06b2c1ac245aa255b1ced2ebf2d5513bff62" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792949548-uHSmSQtYd9d2152l52kK", "proof_sha256": "cb6a9ae72c3203929ceb1bc81af5f55b323857aa40eec9c7b51896a3badda6d7" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792949565-emP7w6cIV7aI4PuqcJD1", "proof_sha256": "dc597d5a9a83e914bbc8e73b8829d5712271ec4d041779e199e0f74edf4853ef" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792949587-K1YSgtiz47Z8fJQaeXkd", "proof_sha256": "3e2e23b0ebcde6f6d51a9c4635abf599ac302df07625bd46cfb8b221b0d09cd4" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792949602-2ED73sL6rsJ35cYtkxSN", "proof_sha256": "6a1dce8aa4845d64162ff5e6259f375373e00c942301097aa21eddd8f8e31489" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792949613-1CQO72wUaiDbdr3RrNSU", "proof_sha256": "ffd01da9dda94f36f3c1cd7d1179529b53433913ed3f2c16082eaa4528cc7ca2" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792949626-7H4Dg2UvK9lYjcg2nxVT", "proof_sha256": "ad8afbd1a6152c8f748e99aecf7a3df78ec9f3d19373cdb5cf076461268e9c19" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792949639-ZzN1ZE0xxWfeExsVnSYB", "proof_sha256": "26eeb5fa7f10c0adedb130e5e9494c5f3feb09523e99f6360f4ca6df1759b322" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792949651-83neFMvVfDR1nfljjQ0r", "proof_sha256": "41639396b3111b005e69200a584e05bf270d2642b86c230d99a605a5993d80ed" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792949675-c8fZtnqID9cD26selLNv", "proof_sha256": "357ae8e06a8247c6b896c6f545912a7476bc57ce1c9634d84f711a18b5cd58bd" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792949700-ho4FYaG7pHQ2RRHk7h3p", "proof_sha256": "ef117b6b3de0b6d1b9b8e3fd5a20a5bde01b1f9e1bb3834153e5c49741a1bf95" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792949723-LrGB76UPV0RYNvbhiyB6", "proof_sha256": "f2e9c004e312c5cffd63465c2526f94b812a13895e0257d566f49c0c94cf647e" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792949749-ZPL0wlMUha9gLB0w7f80", "proof_sha256": "a8dd0eba02993ccb4934f2903566b10c8841c04e3b26c1763c3cf4c0db2f63f4" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792949770-cPcqkCi4JBBu62JXkMQi", "proof_sha256": "26f223db068b128aecbbccb7409c16270bf23e1b62bb67d803588626963abefb" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792949783-zEIgkvUR0P8uColzeYOj", "proof_sha256": "e503057815583590a726afa675af04c1e97a4190bbe17a4d20fb74cca7e7a189" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792949803-UKYGzTpZXSPicYlva3X7", "proof_sha256": "8fcf78e7e20dd25f087933b138b8b574b13948a7378ddc4df385db14c716069e" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792949826-zJOXrrpUXlBATPvG901X", "proof_sha256": "3da89cb86700a564889c06878edcf120c3e92d5d5b9cb85d6d2991179fafe2fb" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792949840-gpGUMENHnMbrjianYKYT", "proof_sha256": "569d427bcd0ffb43f99c0c6312e4ca81955a123b8e7c7547a180e71871954886" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792949858-oaUSsFY3IEYb5PnOxALQ", "proof_sha256": "a29519174a1dfa34fbd66ba54bf8708dc6b44222ed2eb3a3d8fea60a098ef9e6" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792949874-ikujW0EZsU578955KyVa", "proof_sha256": "904a5be61cdb94833f77ffee41b7ba91ac1c7d7719d705c7943039c5b7020134" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792949891-T1VNaIpcyHgzCfyzbO6G", "proof_sha256": "0009f0afb21825822088107c004f28426c704be5a48a8869cbe8ca063cf4ec0c" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792949938-eexsR5hu2KAt8ALoy2iY", "proof_sha256": "a756537a904f5d8a6603efd78d708f67c3a675dff832f49cb607504a09142c68" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792949949-pjmswpNFJhajZwRnQlt2", "proof_sha256": "a7ab3e2f1e0cf27f469adc000f07e19595940639fea824f7db823a7e7a187bb7" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792949991-YH7rUYIgp1L29e8sVwKP", "proof_sha256": "e417d99f936885427c29f2eb8f35c814784079517ec388ba765f5e92e0a6e88c" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950021-bRgGJWHaXQbSRF4ovMf7", "proof_sha256": "b43c959885eacfaf13384be75e9eddb0ca821282577b7a421eebb76d3c333ec2" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950048-ZZlZ2NWDlZHkqdpgZEs7", "proof_sha256": "dd88f969d8295d44b305cbcc6529c34e0ca2d05b5fbbab657c110a84ba1a3e19" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950076-gsT82DPkTTXwSExqvJaX", "proof_sha256": "efc1b8e5127543f3a1a4eacdf9e1971d09f7b1c7684cdd126cb323fb10b31ef4" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950100-ZP7E3Mg3u4wIdlpE7B4f", "proof_sha256": "8dbae565f798a73af3bfdbe70e605a6d7f9681013c6b1cd039ab8cbeaf146c90" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950124-rvkk31ep1Rz4ckiOZhnd", "proof_sha256": "cfed23812433246021bdd7e5dcebb2668a55edc4c52a7ae779c1bb9ed11bc387" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950168-At1GTNjCWseQRHJmvjxm", "proof_sha256": "1eae3b2fbce37f68806581e3c242906164373f4650a947e9da611ca8507ab776" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950216-HZsDSzEKmbt3J3r9Uelf", "proof_sha256": "862f85e2dcd68b7575b27e1218600f9cebd579a4d8efc8ef03a8c9732c4d127d" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950256-1HaXtCkLReudMLFYovPl", "proof_sha256": "149a1978460d23167d1dcc236da7ed8c7f620764fa7c16b17de588e938b7a559" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950277-mDDk2TrlqpHbxUs1EpOc", "proof_sha256": "00237985f7d24556aef09e7b8fe160c3d83e54c816eef3d581f1002cfa53066e" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950321-A5oPk24t1qV79oU3tINU", "proof_sha256": "e1c9eb148e6c45fdc9bee032f310adeda4193a6e31035590e0cf0c8da0a9f4d3" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950360-epBEucb4QrnNqPe8Y4L0", "proof_sha256": "2e3c131961e90071f1a5fdd0196c52e0a7fc21643d6afb9548c281cb46aedd00" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950392-3Hni7bm2ljoIQLUz9YkO", "proof_sha256": "be91e53c9a92783a30f89c94fc00a0616df2884d20958b7e0b5f221eb7f629ab" }, { "model": "anthropic/claude-opus-5.5", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950411-NcmPAOeENOMybie6nt72", "proof_sha256": "8821131efe8d93f6b00929f4404dafd5f025581ba55e422b6b4ff597704ba8f2" }, { "model": "anthropic/claude-opus-5.5", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950425-gPXUquvhBqoDGOO72yBz", "proof_sha256": "de6593ae158e5b374ce4312b0675e79a7ff7ed9438675425e077d667e37d9633" }, { "model": "anthropic/claude-opus-5.5", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950442-QAabf0Juv8MprvksorGS", "proof_sha256": "7d494c0533175bcd317cd92d1aaf210100f902c1c319554e4fed006ee4596cef" }, { "model": "anthropic/claude-opus-5.5", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950457-SLhHqmcMwFNY8izIKmOW", "proof_sha256": "417a0726f637cdd92f504c1706728e5d0e43da0b5767401c33a34433f9c2e9d4" }, { "model": "anthropic/claude-opus-5.5", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950473-dS1jZQ6Vise9GBSLj7Xm", "proof_sha256": "385ce37a4df8ad0914314299417ee4c82ca6a680a38934f50b03edb979ba45dc" }, { "model": "anthropic/claude-opus-5.5", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950488-ky6R08CGDuJRqcT3V9g0", "proof_sha256": "f0220e7ac3ae093f09e14b1adccff4409a965dd2da2f19af3d9e53dfc8653602" }, { "model": "anthropic/claude-opus-5.5", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950516-nHnR04Me8M3R1KuoNhFj", "proof_sha256": "c0cea638322116c692be3e4a5f46b2314413373f3ca2f1f71fe3eaa49cd0cab7" }, { "model": "anthropic/claude-opus-5.5", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950529-LjgJN0chzjX1da3Noqmf", "proof_sha256": "8666bc4dfddc680995d28ca480bec54c24dba345dac3ad51ac0c04272b771102" }, { "model": "anthropic/claude-opus-5.5", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950551-iWWXOra5AH61YnKS7dHL", "proof_sha256": "d4a1160401d3952289ffd38f7f8b21192155f9c16723951494cb09a7ee3afcbd" }, { "model": "anthropic/claude-opus-5.5", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950563-uROyNgEF4CAAAzBQnuFN", "proof_sha256": "b34b8c9b0962aee13ded48849c9594a1891e176956b5d48dd5e111508a6a8b00" }, { "model": "anthropic/claude-opus-5.5", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950576-dJu3NyYwvXc9KZfHOzwE", "proof_sha256": "b27225668dc0c2e887c41ac8ecfd811e39a6d986c3ec76de82d5ec962c0b26f8" }, { "model": "anthropic/claude-opus-5.5", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950599-qyGUyRL2i4bJ2AIAqoDA", "proof_sha256": "6ea7bd597e2b2b09e9c36b9e40dff5b54982a8f1c6a64e7aa40f2ec2c11a610a" }, { "model": "anthropic/claude-opus-5.5", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950612-tmh0kemdzYrel5cKkXIf", "proof_sha256": "0d4014d6c32b6ad4e19ddb895d728197d4875508f90efd9dbb23061ed85c385f" }, { "model": "anthropic/claude-opus-5.5", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950624-HLqIJFvxOVD2i7B2mhkJ", "proof_sha256": "07da961be31563336422f9e4fd10a7b63091b4355b050dd26bb164e5c880e742" }, { "model": "anthropic/claude-opus-5.5", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950636-KgpAQmErU7Q7mUw0POWR", "proof_sha256": "31880dc77389dd7d3a3202e6f86fab72b51f4de1cac8495fe69b28d834878594" }, { "model": "anthropic/claude-opus-5.5", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950647-VbQyhpahHu8NiXsQY6Tg", "proof_sha256": "4eaa8653dd3ff3969bc15a9810a6c3dd532bb80d3b6efa81f18e4a857c1e5958" }, { "model": "anthropic/claude-opus-5.5", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950656-7Yhl0hz3FN1ba2rwqWjX", "proof_sha256": "d0e0489f087aa6ec0d25efa5c61b51e7f3a75567d636e853eace71e7cf7fb3e1" }, { "model": "anthropic/claude-opus-5.5", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950675-tHUs3DUJSvAFHzfqAiaE", "proof_sha256": "2b2023bd1a27490ca3311d249570c2761a9c03fd03e943b43c351e7afc447b06" }, { "model": "anthropic/claude-opus-5.5", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950688-8SgeJioQrJwePpsL3qE8", "proof_sha256": "2dd8b8714580e7a23c21f04ffb5e210aac18ad3b910561e9355e6d068a16e4a9" }, { "model": "anthropic/claude-opus-5.5", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950702-bdUex09920N2Ga6zVwzd", "proof_sha256": "6e8015803f45834ebd13fd4527f50cc53d26a76eea3c2756449f4775a1bcae25" }, { "model": "anthropic/claude-opus-5.5", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950713-IOI9W8zTIAQuTHJGEdAN", "proof_sha256": "f752f2642fbaab9adbacca603cef76aa0e6c15786fd6466dd54ff5e5f94d3962" }, { "model": "anthropic/claude-opus-5.5", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950727-8hvp7Yx4JFerQyZPudn7", "proof_sha256": "afa02a54227ddb490a9623703cdfba6ed5b6953c39057662c95f06a378932bad" }, { "model": "anthropic/claude-opus-5.5", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950739-KDPmdRzdK9OCjo7L88Cm", "proof_sha256": "80476fa5d5ef5a1bb6d8ae04cef47fc3b117cfe96a40e77cc92a78263eaf293c" }, { "model": "anthropic/claude-opus-5.5", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950752-uuaHpUb1xc2Xb21wYGRQ", "proof_sha256": "3b7d936d3a5159bc666592e40f4c1b1e9faee5307d6ad96ed7d7ebf12197e910" }, { "model": "anthropic/claude-opus-5.5", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950762-5a8FZTL6CFBsK0o0JxS3", "proof_sha256": "d8f8a4021383b14678c81b6b40403cee334be21eed1474c03497a09217a957b3" }, { "model": "anthropic/claude-opus-5.5", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950785-rZjJGeQY95fc1tgGr7QP", "proof_sha256": "62d548ad9829e76597f2fe6517ed4839533a7e3d76a9200da35400e36f0b3dab" }, { "model": "anthropic/claude-opus-5.5", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950805-DoIgb7f9i5fyz3bhqlRC", "proof_sha256": "dd1c7e18748540f102eaca30e7acd6e259886b2a5c44337f38d6cbc555298223" }, { "model": "anthropic/claude-opus-5.5", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950819-KO47vFh3dMpGTEDBTbSR", "proof_sha256": "b0ee40a0d87e4914bb291704a7638c1dd516debaa8cfaf3fb26ed0b50d659dbf" }, { "model": "anthropic/claude-opus-5.5", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950834-fRI9mxLey15yDUbaKdV7", "proof_sha256": "23666f99c52e39a13b200ceacc00b857b737e8941adc3268a51f72224697df4e" }, { "model": "anthropic/claude-opus-5.5", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950844-Dsn7jbtTnkyxiXh3a3xc", "proof_sha256": "7b1c8e23268e1bbc17bfcb2da35462e276547dde1b86ec2a17203b4d05279aa9" }, { "model": "anthropic/claude-opus-5.5", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950858-xuBJjEW3IQJXRtRifDtB", "proof_sha256": "fdb0811a50b8f9717e49a264cc1ed012f5e6b935361edde4177337d210c00773" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949253-L3cvq95S9LB9ysdhecfO", "proof_sha256": "3bd64d3597f43dd75568026c69d89f6011ccd0fc900d61916b9dd4c757376958" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949276-w64X5iHZxyDSkKgILvNB", "proof_sha256": "f281c9a280537a15a03aabc5305ae74c09d4d706d07f4b9d977879d34793294d" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949300-xizzvoMFyk0lQi5ElIKF", "proof_sha256": "5253663b0331341aad794169786a95058f885d48015f7b4cec38981dc12b7b7b" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949313-f4wX1Eup904GbS4rQw71", "proof_sha256": "cc65c8fd7946f8d7e45d935a90d47fc5bd73f0359f5174b392f74c085ec03e96" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949332-ctbSfLLJfK11Ic90r00p", "proof_sha256": "52fefeafc2ee07d9bc6288d9d5038ff0deb1b2976478955fba4fb5735a1f9a0d" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949352-WuEVgeWMC4GltGDmCWMk", "proof_sha256": "94e011afa7bb31603dc6f7cd74f6a1849a06faea04f44329b551847d426d808e" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949370-X9EasagtXjhry004g9e0", "proof_sha256": "818523892ce8b3531ca9f88803aff4d39150614e251d2065f1e04a6309467f0b" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949392-4oKp2FFMp9DNjyFBIyMh", "proof_sha256": "7d5bfe2a180b42b3a278d0cb065b6912f8007d5bffdafb5ac1d6164a23bca5af" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949411-rnwJcPh2LXBlzJIFVPUj", "proof_sha256": "e4ce75705467bc2716c8db7369293f4582d5890f071c2699c1f4f6d04f48c9bb" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949426-ZI2tLyJHF7zAnQT7SaHo", "proof_sha256": "0bdf7f345d941ff825d901bc56ec85dddfeb42b1d06b63f49c6a7b381f4d0c47" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949449-RmtCL4VH66QSKSLGbZpW", "proof_sha256": "17655750d638c04fe3c50e4fbaef16b0aa9943bc705fd351e3cfa2228a512438" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "$$\\tau \\approx", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949470-IeUT2AfTfCWvagPVJMpc", "proof_sha256": "d4c884501bc792b3ee30d7b0be0d49baacb40b332258ba747482ab40bcaf6461" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949515-ERPKSn9LTSG7AlEGsbWP", "proof_sha256": "9300549d4f15bd2fee1698738e2348d0a458a7583ddb492814071bbfb796e343" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949535-aydfdV2urQ8llrUXfyPP", "proof_sha256": "c78cfc556d170ed509409f026cddfcb52128081706e3fcaec3800aaee0198e7f" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google AI Studio", "run_id": "20261025T172733Z", "generation_id": "gen-1792949549-1AC5a9TasT8AxzxEJ8Pp", "proof_sha256": "65562d6fbc4a425e58cf873a75aecea862b0788b2e3fd70c07eaccb653d0fa12" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949587-mQ749rr7ST0WF8TOe2XS", "proof_sha256": "63b7d80c6511e0b8582d5620cb303807c0b5022397fb6c0c60c425c89750d918" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949616-MPNcVVzme75AZ1XWkx8z", "proof_sha256": "5106b665388c333005c05cb50a7cb45554ce0118f3baf487b774239eb8fd22ab" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949630-55407noF3eZRwMtsHe5c", "proof_sha256": "0afad1e4308df117c2dcd195d9edbf02f5952a8f92d69df5b77c1fa5e4887adc" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949647-Ksw9qbxTeJqGXuaF6wcO", "proof_sha256": "94ea395c0532001d9c4556e5ce134ffb4719e20ceba37accf674af11786f6cc6" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949660-0DMGZkwGBZCtzNqxWnj8", "proof_sha256": "fd297cbe76a3ac6e43cb4dfea16a7fb7ed75d083d23172e8e1d73a81d46aa9c1" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949671-Jxbdi6toV9xyg1TxwRfU", "proof_sha256": "5674960f387854ca61a35aaa757f83d7547f7bec9bcec4a08e5f361a6e14045f" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949680-BMtLoHWcRci2BjOzltwO", "proof_sha256": "66be7c7243d280a9ce16a24114a5d2a26b71c9054c2bd01c9a4b197f76510e77" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792951013-x4HsROj7iXKoiFaHiDFd", "proof_sha256": "af83dce330bf19b85318ad3a9f69277a97a9fea961d69122e22dfe39b25fadfa" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949698-5hYedyrRXyjyCR5xYCKJ", "proof_sha256": "1d5ebe658b186c4097f0cba0f929c74ff9b648435b46956bae8711f6b7cdb4f4" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949712-MpPRfdKfYezyVqr0Vy7R", "proof_sha256": "ec52cc003233c82cae2a7dfcf75d59d9a7abc2fc6ede748dd4cb366f04238b9d" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949753-ZZxklnMhtilBuO0qSi6j", "proof_sha256": "3d980d8e3648e9b361fa42230bab58a6e1107d67aeccebbed822c761f5e03d28" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949783-p4DUcoY8oWR0B5rvmHDX", "proof_sha256": "b28fdbed691c6625d64382726b5b45bf8ff5fc3186c867c2872c80a7b4f33459" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949810-M92mtkQKZYTm4pxFvP6k", "proof_sha256": "76445788f5ccfc8ebc22c5e99cd3bc1ab530f01a9b8e4baecdf177902c2bf335" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949826-9yBl30OFPUCO4K5jaCBa", "proof_sha256": "41f63e21e56723b50b14f490572f2e275c5378f6ee7066b9b586dd58eefa8094" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949850-MupxcVJPhM8YVvw251ZB", "proof_sha256": "9f8d75ff8b52fc4ab7f1bbf6a275c4f03aa77b84d63557975f4a216d986ea974" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949874-elC2SBCIir95JanwoVGk", "proof_sha256": "5fb16ab2d8d87f0c12a47b73acf3e772a1015e2b19599611ad5213c4f9e9d492" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949899-Zji4qXdsThSJh3VSbwQp", "proof_sha256": "7fef4b0666b8b7ef87a3293942dff2244c7be7459b067746824b7753295bf868" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949915-dpbbzxU7ESmkQCQcQKnR", "proof_sha256": "93a6aeee3e966bf9a495fba5f9e7f273a8f2cd66b4fd1625dd8a687a25a5e3a3" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949933-P5ccX6yllxutuCkr4sTU", "proof_sha256": "d0081e445b92152151a8b462fbca0a4f0ca3b4c7560e5f85e2419e6c9cc57522" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949982-d07dv5sMubuV0WyFKQOi", "proof_sha256": "64181adc86312674dbc688764da065980b1ba5bd5297012c77abf39434e5a9e4" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950027-u3FsBPzbEvfwEWF55XpW", "proof_sha256": "2afc587ceb3920737e567beba89a0abac473a5ff8ade50bd360c562beac1ef2c" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950064-ObQZFVJxLTXnmzy1ELzE", "proof_sha256": "683cec17367f8aa2bd17efbf7093c9025994da33b391cd43874fc74e107e55a0" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950074-5UuzmcfZ4bJlSGgP9TCq", "proof_sha256": "e5fc97620f08a167b8b3e9d7c75ea375a60a2c3366f3abad4b8650a1aaa9860b" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950084-JXZn1S74W6MMvilD9j4v", "proof_sha256": "e5e79a7547634e33ced39d40b8bb4c9d8707e3e944972d938d5f092fc7506a4d" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950097-bzymaokUW2ODIiMiIdyx", "proof_sha256": "867e5a67cf1eb71761d7947f9646efd18043f0fa0779b05c09003c9a3ebed2f0" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950116-XNImOtxz6JtD59645gYw", "proof_sha256": "1b0cabbcaddd22839918dec23b678f52d249581fb07ae9ff7e4317d0ae2fb2f8" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950158-oPIDypoGcXpgesAnvCuf", "proof_sha256": "c300479feca2f8f7aa01f8bf98dad7905615331d1fa6282375ea380600ec4752" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950186-ECtESKKkVi5Ln8LKHmhG", "proof_sha256": "331dd9e008fac225cd934a5e8150f29c17957508f548efdef65d06fb940eb9a9" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950230-LvLqR34eOyGTIFBrMjOC", "proof_sha256": "acc67caefdaebd6e71d0bd0ec0198c5c3a4802b811653e5b196af32be8b868d4" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950262-cFrcUK7vj6ai65YtBMZZ", "proof_sha256": "197f38a1f89c37bf75b4b435d9760046ccb47899399d55b1f7b6b147a30cfcb3" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950281-TEiTJgzt38cewYUjZXYj", "proof_sha256": "936c2702bad3cc6fea160c09fa81d7e11f5d443a6c9bf5fd72141ec319df410d" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950323-dGI0szoP30dmkgNayvqv", "proof_sha256": "109fe9179afb102cf4785942e39469f080989a4b74d080a0484befa47b02b544" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950365-8J9PrcgL9ejApLTmJYK6", "proof_sha256": "7c7335f64732298cd39c7695eaa759d285d0844608d841f9e3f0418124058578" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950392-Gj3hsq5e9aGPfvwnOgsj", "proof_sha256": "8d6e96a41302d8398d0b12f690524ae797e79ed014cbfa8972a1a3802502d5dc" }, { "model": "google/gemini-3.8-flash", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950412-W2Wq8S03plrWN6DLSNXR", "proof_sha256": "7e1a36ff381cd7f5bcf4594f8bfc03ca0fd235816cd431f1a3fb3bd91d16f330" }, { "model": "google/gemini-3.8-flash", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950423-DMdwQ9oI5Wwm1WlxCsfn", "proof_sha256": "d3e4c9815cc9d92adf7e11edcedd13eac4c123b690ffc2d2365b7d801deb3b3f" }, { "model": "google/gemini-3.8-flash", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950437-OwTCuDGdBdE5R5FjfrIb", "proof_sha256": "bd30151571cd5e33a2b5dc2b2d46911375ed9893ad7993ec5b32bf85486c184e" }, { "model": "google/gemini-3.8-flash", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google AI Studio", "run_id": "20261025T172733Z", "generation_id": "gen-1792950449-UJO6y9v57JujRd4ofjST", "proof_sha256": "1841ad0a7b6ce6d0eca9b8dc2dc516314999f9db643c4ed6498c1c2dc124fa39" }, { "model": "google/gemini-3.8-flash", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950470-vnHu7yLsxfi8mzMwA3Ax", "proof_sha256": "b09b203c655afb301964a767f8eaa5245075f984ac53e6b615d3177cf75d07b6" }, { "model": "google/gemini-3.8-flash", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950492-jBTRIuJr43TgviRm8D9n", "proof_sha256": "e68132f6e2e1a441d0c72a1677978e4b25079d1b757578552093bea7aec9768e" }, { "model": "google/gemini-3.8-flash", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950506-4QxtoxNBnU5ZMxdOgiMT", "proof_sha256": "face515e5219a4c3c0f0697e89003b639260b823d6862ab3c1e06cb018064634" }, { "model": "google/gemini-3.8-flash", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950525-Xaq9EZR5fmqJWtL789ph", "proof_sha256": "225da7c43a012092fd377063492d937f0a238ca260b915ad71750c9e60cc970c" }, { "model": "google/gemini-3.8-flash", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950536-lyoniTOaCO4sTuZvVDbT", "proof_sha256": "08ee2afe2c76fe359e26a018df1e9f6830c8f85db8336ea4f5e76cc3df4a4fb2" }, { "model": "google/gemini-3.8-flash", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950555-p4iE9oP8ffRyXuEfe3h4", "proof_sha256": "a3b1d5dca50271d2e0187a977da0ddc29c2d83cd58a82eed5147f482bffa2f06" }, { "model": "google/gemini-3.8-flash", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950564-QD9iHxYV4fRWKnKevbxm", "proof_sha256": "b8de90726243ebdedf2fbef697a3735fd5d59238e82f17ca2cbd0a91688491a8" }, { "model": "google/gemini-3.8-flash", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950573-yymJMaZ9N6HQdqkBujcI", "proof_sha256": "f20cdcf586ff0b69ae84b4651933fa19df7afec921e143ae06efbe808394ad5a" }, { "model": "google/gemini-3.8-flash", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950592-7S7zdUGy3JgXwNecevIL", "proof_sha256": "3ff54545652f3ce4350ad72495f5e59c73e3461c008bb000cbc7c4c709d7ad15" }, { "model": "google/gemini-3.8-flash", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950600-0psvtbpqvMOiSirOwEUD", "proof_sha256": "9c86af3399cd014bc29526a669b84e5833e4d4439717e4238e5d340f16edd1a8" }, { "model": "google/gemini-3.8-flash", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950612-OHcbitVMsFXkdukoJ2NE", "proof_sha256": "955e71354552a778e1bc97d6c890425cc4cb11ace6712009f8b4f9f083c0cab3" }, { "model": "google/gemini-3.8-flash", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950621-cBRmYTLoqmuqcLmTxTni", "proof_sha256": "27fa831047f5e29a0dec13ddb4066427f6e3b02bf18058ee0b25b163716c9505" }, { "model": "google/gemini-3.8-flash", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950639-xUHT7MVxpedYxpXQAYti", "proof_sha256": "ed1ef1e5c1d78f20358d3fc657cddd247eb89af83d7cc002baf88bb971fc2eff" }, { "model": "google/gemini-3.8-flash", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950649-UfDa1gAazLO7zU98ZvwG", "proof_sha256": "d59e0b9b6621e3ad3516d7a9e81f272f4d1084fc0601df9eb2a5778ae1c09ee3" }, { "model": "google/gemini-3.8-flash", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950658-8fdNZhr2CFgpr3iBWNTa", "proof_sha256": "3993b583b8ddcfec6dd03e192edb2a3ecdc30f37ec844d5026006f14d82d9d3c" }, { "model": "google/gemini-3.8-flash", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950670-Vc0NxlJ55WaVzb90ndhS", "proof_sha256": "5e8fa0ccbfbaca5e861fec7925d222257b1707f9dc83a8f4d8768604486c5153" }, { "model": "google/gemini-3.8-flash", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950683-DTtI2ARdrbtc30ja15un", "proof_sha256": "7c0327d81a2608a91370a7d69d21d296e8fb6676ce5c1b9a0d8f2d17211f450b" }, { "model": "google/gemini-3.8-flash", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950692-P2j2ol4cZ19CZsA5rRtk", "proof_sha256": "d9900b7e010a2cb5a8e4802af2918275fdc3823e7f533ef03ab61aa8a0b7d89c" }, { "model": "google/gemini-3.8-flash", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950701-Tc5EBDok1f7jhOpDL15v", "proof_sha256": "3735889ceece7f7e3eddb71059388d538b5bad85af874273c4b43f4ff38794a5" }, { "model": "google/gemini-3.8-flash", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950725-vFPXQGJhKVeGKQndS3Bi", "proof_sha256": "8ef1414bf582a5a171b89501c348d7f346ba04045a62134806479232c0ec8156" }, { "model": "google/gemini-3.8-flash", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950737-432ftg3WB3ogxgkALDpn", "proof_sha256": "5865c0ef951b998f453cf0ea6f52aa84cd46abee5b4ebf20a657c763d3267b15" }, { "model": "google/gemini-3.8-flash", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950752-ykA23mHYZbl8krqyNSaz", "proof_sha256": "7fd9790f5da2319516a77dcc904fa777f2d645fa84ce1386cf858a55ff901eb5" }, { "model": "google/gemini-3.8-flash", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950763-No0TT10OBTxXJl1JS0oQ", "proof_sha256": "c8dde4886c7a7a297616fca00d0b47da7e0dfd729e2eb8e28e07cf9ce631061e" }, { "model": "google/gemini-3.8-flash", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950774-BIZP9kgtHpxGafD88TTo", "proof_sha256": "9e44c55da5826d45c3afe8e1ec731e29330541a4477495825d099f20591e9b4f" }, { "model": "google/gemini-3.8-flash", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950786-OW7q4bB8sHf8ESLA71bV", "proof_sha256": "02aab4dc2f2d88a12ab58e02255a8d52180aa435107515e6e381e92da3320d70" }, { "model": "google/gemini-3.8-flash", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950805-vX9pCat1Ipf1Nt4O7uTP", "proof_sha256": "c2beb3a0a30853bfcd60fa174f519fcd7e732a142d8e26d555ae0b8c3c4f8019" }, { "model": "google/gemini-3.8-flash", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950823-HGHFOoNYRlg88Wwc6pKA", "proof_sha256": "beb4f5b488ff5a0ec50f1b9bf9be3313ec57e31ae3e354f1fecddbd56205c724" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949253-Gsm0J4Kb6w3HqKcX9ewc", "proof_sha256": "531c64a7e21f8f342b459ae8bd9ab7d9eab4c78107e8db5b3b89bd14116e8893" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949272-0yFWkcGhkeafbNHcBWv1", "proof_sha256": "74855bda8296c9e07cfa5e0d8a96d25cbda89cea348dea222268b95ea8a5aefb" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949280-hm4o5Vv8uSTZKXsC0ywf", "proof_sha256": "5cfedbea7afde9cdc48c466036d35e6328588e6160f1ab67a7dd1881ab58f13a" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949288-rQpbAqvqQ1J9ylYBOdxW", "proof_sha256": "0600935889f15435e0407203106fc453f99f74bc641b89f5534607994570e24c" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949312-m1HWTh4CK9bQSdwWVqTU", "proof_sha256": "f1472f6a21037027dafe03264769cb7315a180112a69f3a5ca0a2e79be939b9f" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949333-Hfx9MHMdq1hpqJpLmGG2", "proof_sha256": "558123ca68d17f1db028d0d8267b107f33e206c3561deef9008577991e17ddc9" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949352-eJMiqEXnq7SkAqA8n0SR", "proof_sha256": "467c61f9d8a4142e170d688245f39a26e038aab7fc2b7968d0aa8861341e5dc0" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949360-yrqgod5Ofp8enrfkpT5a", "proof_sha256": "54f3797b643b15d9a3902aba62112464ec1fa230852fb9fbad5515a86b703d42" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949370-0g8PcwLq2nBdhTjZW9Ia", "proof_sha256": "88a92b654e0e3600a985fca1e4401f05030da5c80d395b9950f6a70dfe68f221" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949378-IQdaUjECDxYCrekI30oV", "proof_sha256": "340ae32db1ed3b7b08cc0ad537a2b60bd189899b04c54fb8fd59daacc5bbd325" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949408-fayK77k58SVQhnmAySJV", "proof_sha256": "f98a86c7fe2dca25109eeab5c84a30d912141411abdec1aa3a27c2d1a2e0edb9" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949415-VCws6YOBIBKwnISDiJ8O", "proof_sha256": "9d26ceea78c599cbdae998a9e3a676259056552943ce58987ed6dcc15288a916" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949450-mQMyAjGlXGk7sUWjW2ek", "proof_sha256": "311ee9d285df1f28f39bbbcd8cf19e6086907db3f26fcff0266cc32ea2ae597f" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261025T172733Z", "generation_id": "gen-1792949467-2VZMFjunIHoZQH5oI1pj", "proof_sha256": "60aa5667ee26ad5c96645f7fce3abe56c63a21697b1c12089b45064d92a27ec7" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949498-YnCE7qiDows4Ffj29Bf5", "proof_sha256": "1154d7f1aa85743a7afee3d2ddbbd86a1ec2be5f01797e4a91345b09882d6e99" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949517-BH90WjT2RIIKl0EcQmow", "proof_sha256": "e03367138526dc76b48ad850dcfe0fe849a57c15728af63dc67a3e6b23110837" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949534-fUqH07jNWDZ8ofJMvsrY", "proof_sha256": "5565aa64b0be21cd9593fc1932b8856faf4e2d10aefc40df636ac075c584a307" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949542-5aRPl79bdAiF5FniuviE", "proof_sha256": "d3291e36f19ae7c4c3bf661923ab717e58bda861b4416bb42d0d860a7f39de1f" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949560-6YV53qomFhYdUAzGUoLT", "proof_sha256": "1b9bb3413d40d287d8e89ac1f0b2f08043922e13c57b98bfe99e89bb2dc35d31" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949577-HxHp7TDH7l0wFWoiTSrG", "proof_sha256": "a9ffb4d3546ad3549bfcfed4972eda529e522fb49df2ed6ca017f0783f6ac961" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949595-4mNNwF4NP1bFzYt1YnYU", "proof_sha256": "14f57f902587aebbbe6b5e59ccb641fb715cb6c3dd5c57852fecea817ad7b983" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Baidu", "run_id": "20261025T172733Z", "generation_id": "gen-1792949603-WB6IpmjX3DaYtMi4nmSv", "proof_sha256": "42d9d0e0cd2b3eea52962b850bf76d3e1dfe60c63198288eb0f161257b3552fe" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949612-XQUZYtuxuX78ehZ5OH7h", "proof_sha256": "89c446bf60d5f05b6cce5f971940391e00065a06e1e2c6691be876520cd4add6" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949639-ehqVux3FYLkzjyXWOR3z", "proof_sha256": "0b6fca8a29b581889972140c074dfce5077740120c6e0a3e0ad1f72848d5ef0b" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949647-G7l4KPzabVJNZNbgKlQ3", "proof_sha256": "fb6637c60233a07f857217425d73dc56993fdd5034d3603744f350b65899a778" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949658-Kvj71p8un6HGqgR5h4ES", "proof_sha256": "20c51d23c8d032a9fd7ad08b7625d95fb0b1727fbce17f08e94d80f55cda9c99" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949671-ZZWQvl52iNxFvmukxGF6", "proof_sha256": "b60fc79277cc6155b1e9452d85005e094b967f2fead9a1e6d2d5e157eb2c29a0" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949679-Lxp5hSnEpxWmEXnzzqux", "proof_sha256": "6c8f64c5116c6143f5690b5a8d052f19e68aab52313765ca8ba7b24fe22a1f31" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949706-pMXSE7ANkcKlzGaZxNtu", "proof_sha256": "8a824923c9d1214b425af088ece8da1bc134a171ba33be08401d52890101da6d" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949717-lrGZ1K4QO4L4wKCdZAkO", "proof_sha256": "b195eb902b0a08b4ca9ac2783052a53ca72b3c57d24bf3ba2ce52e38f7444614" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949730-tfvZuRMMQ3wk7kBFOjb4", "proof_sha256": "a288de328ff77c23c16b5c686694a0139090441ad3e0efd3d90af8956ce996bc" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949744-RbaA5KptamYLJKAYlOvY", "proof_sha256": "31a7bed2619ca83d6c7e3bde4995abffea6c1ad4c49c26c1ce523cc1674515dd" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949763-puxnKmSsqu3zQMKu6C6o", "proof_sha256": "d777f3495268cbdc1cce77a6aaf1f1351d5c25da296733d098435f7de58695ea" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949782-JS4fChw07MQUyETNo8ee", "proof_sha256": "463ab36933b682649f31d7bfacf92fff67e2eba409bfdb99bf92b26552126b13" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949790-OIZqfBtqv4ZjqLuiElUn", "proof_sha256": "2f81806e834f77871416f559b227bafc679e88bc237a83f2e505c66f59bb4899" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949811-lW2S354SAG0DL49TJ3xv", "proof_sha256": "d4a394426e578f54e0841a5509309d9b7515e4d64149ed042cdad6fac3096c5e" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Fireworks", "run_id": "20261025T172733Z", "generation_id": "gen-1792949819-sEJyKqqvOACqSOrj0P5Z", "proof_sha256": "dd02e641db316bb0fe2dfb328e62712ccf9f713b3d3bb6e4e491511aafd91dc8" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "PrimeIntellect", "run_id": "20261025T172733Z", "generation_id": "gen-1792949831-bdFjRn9LD7OZvtCP6zha", "proof_sha256": "1702e3d3372d4470c9d99d6c73f06750bdde7f1d8ed5fdc14fb9baccf1f63713" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261025T172733Z", "generation_id": "gen-1792949840-nFi2Ylo83IQ0zPvIVDyZ", "proof_sha256": "69bea5b02eb5deae99ee130a8cc2cb53bbba8d5692b86b15b361eace96b40c50" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949849-WR5KaER60pyPFJFpPQuJ", "proof_sha256": "a7882305bbc1663c23cfa4b389464b793e97a5c9824c700fdbba6ddcf20b63a4" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949856-caYvZRO1Uz0nUWn6L50O", "proof_sha256": "e92f3716dccee59fc9f22a6d8d87fadc570ccd088f3e7fef4d09b50e858138ce" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949873-QWsUbWsy0Stp39uRtAAx", "proof_sha256": "d8402e80fb86d522c327e393ede8fa0813a64ff988c7632102fea21d563538cf" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949890-wzCuFEP0Ukt8vxQV1Cvq", "proof_sha256": "d433f1c41083cf5618abc1c29fb50fb8e8a32d5d546bbdc9069c52f89cee2d56" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949908-Hwdk6WMlm83gFtpnWduW", "proof_sha256": "2ef7138cbe46165a32a931d3c9e29ed921ddf4ecc20c4b693ec57ed5a4666b2b" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949939-MJheSXihaNcl8MEWHjL5", "proof_sha256": "7d1d8dffede056b61c3a6499f256b32480f38c95112418f5f627c24352188c7b" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949956-NMBTj6A0NpOyVAKsNYoe", "proof_sha256": "89cbbd02c415f22df35da23b126c5b16a1d1a4a17fb5633741de9bcc4f8f0e87" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Decart", "run_id": "20261025T172733Z", "generation_id": "gen-1792949995-1KJSzvhULIzrD3ErHdbn", "proof_sha256": "3a2707739109922d7653eaab10d3a9b390a491f7f577ad19dff73d0d984ad37c" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Fireworks", "run_id": "20261025T172733Z", "generation_id": "gen-1792950027-vc8gGWPQcML6qpjtLfRJ", "proof_sha256": "0d5d4494a16422e5d674c7081481e3dbc2bb58bb1f34a99eab42c5fbecbdf770" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792950066-MOuXsJBAWIxHDkKPqoCA", "proof_sha256": "e0ae781868912642f187af7b67191d05290eeb3794d03534fb2bf5fb7d5c6b2a" }, { "model": "z-ai/glm-5.3", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "PrimeIntellect", "run_id": "20261025T172733Z", "generation_id": "gen-1792950074-x0EzXM7hnKP8MX1XsnPy", "proof_sha256": "f3b66e7258db67dee0f6a94caff2bfea1adfe6b64bee94bfea26f2a1090d870c" }, { "model": "z-ai/glm-5.3", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "PrimeIntellect", "run_id": "20261025T172733Z", "generation_id": "gen-1792950091-DQpBXWPLDHh1V8XXTra5", "proof_sha256": "517515319dec573f10390e6fb77fcc27b3941eee0b239d056610d1da6846fd44" }, { "model": "z-ai/glm-5.3", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Baidu", "run_id": "20261025T172733Z", "generation_id": "gen-1792950109-Zm7mGdjpmaFX2xllF38V", "proof_sha256": "8c951d57d8acd140d64226aa581677c6c85565d8b7d7cc65bd1e97ad7fcdb88d" }, { "model": "z-ai/glm-5.3", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261025T172733Z", "generation_id": "gen-1792950148-pglv4hA22IEJjUu5o0Hl", "proof_sha256": "4ff21cce021372a8b7f4bd1cf841a4f6114471486d1faa2a604497c06ad5d201" }, { "model": "z-ai/glm-5.3", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792950187-skz3xvpCWv6AVuXS0Kt4", "proof_sha256": "23a7f53c2de146639f6227b5640ea910df03e679d35861cd696e226282ce2fa0" }, { "model": "z-ai/glm-5.3", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792950226-2aYa9hNMx6ElPPAeFFn1", "proof_sha256": "b2046b98a9b70be3d54d4bcc5b546307fc39e4f8185a45657a59393bfbe625f9" }, { "model": "z-ai/glm-5.3", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261025T172733Z", "generation_id": "gen-1792950266-S8xure0cf8RHXgzpF4aQ", "proof_sha256": "06847b85481b5719b2100f951d1f110ebfeafc7c887d31c78248f68c2f4cb973" }, { "model": "z-ai/glm-5.3", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792950283-X9sCxdQKeLP92U8Okrsw", "proof_sha256": "0bdd3c85324a3a37fb94c3596a62ff97dc8e86726297bf739e71a018801f59ce" }, { "model": "z-ai/glm-5.3", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792950322-LVNQu2p5jearlhB1vfnJ", "proof_sha256": "77e83f905eac48d594bd9883d081640ecbbbab338cff0a5a3da8731308525803" }, { "model": "z-ai/glm-5.3", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792950360-RHu8fjUwqDNKM0LR3s5a", "proof_sha256": "014161e79708106c5c4c2fe0496bb13718c3f6a302f9935d13e221cf70360d84" }, { "model": "z-ai/glm-5.3", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792950390-SwdlVY256e0l5YRGXS2h", "proof_sha256": "292b0d0d7c25cf05c88dc5f5e88e49614f68b2a94081929a40ef091d5175d5f5" }, { "model": "z-ai/glm-5.3", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792950408-RvJASYzLsc4H4qUz8Si0", "proof_sha256": "9e3d12b728a85e0bbde7b4c587baf959501f398c0ba7cc893133363f1abe1262" }, { "model": "z-ai/glm-5.3", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792950415-7xms2BKUBRUDKpEYTbqk", "proof_sha256": "49067a63761adc4285905457df801621d5e75ff9b5dd94564c5beea14c4cf9b7" }, { "model": "z-ai/glm-5.3", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792950423-8tMuUycEUU8oGfOBASwz", "proof_sha256": "8957a1d3ea341d1de5647c47710daa4b7350c3a75ae9bcfa5783f3039969423f" }, { "model": "z-ai/glm-5.3", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792950430-k6a2WlkIqAjYDZOva5nH", "proof_sha256": "a92bce178e10843841ac262124d88fdbc3fc80ced4328d3b9fcb0e19fa2a7c47" }, { "model": "z-ai/glm-5.3", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792950438-EUrobMJlpu31reHZ7xN9", "proof_sha256": "a7b166d7083fd849cc05eb78a452b5ec7d26184fb70c6bdf7ece0213baeb7d6b" }, { "model": "z-ai/glm-5.3", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792950447-tmyRnQ5N2YE9m2DoRbKm", "proof_sha256": "273b624a4626de4bce9e80eae91cc33f97854884572e34bf1bb7ebda8e36ee2b" }, { "model": "z-ai/glm-5.3", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792950455-84JcB7GRA8chCdObF2BM", "proof_sha256": "7fb50da356b951287fd8e404ec959e5f947e9f408e75c4375578bb4c9528d539" }, { "model": "z-ai/glm-5.3", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Inceptron", "run_id": "20261025T172733Z", "generation_id": "gen-1792950462-PRns4KfO3Jwby9uwtlRZ", "proof_sha256": "94594dce1e2aa8405bf0589c3661d11b650f09dd8f01ffce73a891818f230d50" }, { "model": "z-ai/glm-5.3", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792950470-76o8j57t4FI8WfoUZcuj", "proof_sha256": "ddcf35683353fa85c2fa6fe4bc5973f76c441bf044df51576ca2bca6a9fba32d" }, { "model": "z-ai/glm-5.3", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792950477-iO9MeYofKeewSj5V5abH", "proof_sha256": "4e2e224fef2f3a4013ec0c0258b8f219f816e5b5ce034e84772a3483b38fdd8e" }, { "model": "z-ai/glm-5.3", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792950485-eIh1sYG3X35gmOEXPmJR", "proof_sha256": "c935e79dd9fd282b7340100fd8f52cf4a251fd6420daedc37e354248d33109cd" }, { "model": "z-ai/glm-5.3", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792950502-i3QCVxErh5MMaopdShqb", "proof_sha256": "fd811fb6f1bdc92019589a1126016fb4374086cc0d5ed9ce2c3eb96b872d7367" }, { "model": "z-ai/glm-5.3", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T172733Z", "generation_id": "gen-1792950521-1h6tABCTBNVlrYI4mnE0", "proof_sha256": "0db8868dc15be772a59e6b6e1be8a6e57b77171a429d0faa38b80ef04070b2b1" }, { "model": "z-ai/glm-5.3", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792950529-01gGu8QFPb8W7WWMg6xa", "proof_sha256": "b5772aa387d5b27db013c11077f47ddddb43718fd3dd85670c12d3278e8f387b" }, { "model": "z-ai/glm-5.3", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792950546-eKDUX69iH9cjAAvJFpwd", "proof_sha256": "3e4b6b432da3769314f8c7b574ec68a28290db5bd6555ebd90805c0ed47e174d" }, { "model": "z-ai/glm-5.3", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792950554-8ePmMxm65qnIrk5RRnTE", "proof_sha256": "6124b480958779caf797874ce78f42cd8724b48c8163aba8c7e823dfeb670cdd" }, { "model": "z-ai/glm-5.3", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792950562-Vbrvg74bWtbjQSSFOZaq", "proof_sha256": "cae1956f8bb0783043f0c12c071e2f7216136a2ef0a0851d40d6cfaae26030a3" }, { "model": "z-ai/glm-5.3", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792950569-FkJx3hNugWtHFYxHamBd", "proof_sha256": "eadf61510e0fc67bc7521ab50cbe2fcec40b435f2ac4ec20215d815621b14aec" }, { "model": "z-ai/glm-5.3", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792950586-YDFV7y9xydpAiqFaPIN1", "proof_sha256": "c0655a901947e15ec7e9be3df0abcef9a6e07250cc197cc45af6a88220f92d9d" }, { "model": "z-ai/glm-5.3", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792950603-fLHYzMyBYclMib9DgsHm", "proof_sha256": "2a2e7ab42d01817ff99abf5d244a6e12b4c8ddac9672c570235bda6ea660a4c3" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261025T172733Z", "generation_id": "gen-1792949253-YTtLP7qFlgfYtYuYIz7Z", "proof_sha256": "bcd9f43d9969ef4e0cc2fd021b6a9d6551b4ad09e55dd548898a47ce2310b181" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Ionstream", "run_id": "20261025T172733Z", "generation_id": "gen-1792949275-QT6uxlHSbrPRQLGV3UaN", "proof_sha256": "45ba49333d8d8dff4d2297b059092cd42856f74e305f1fe69bbe57322fdefb9f" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949314-Oe07zwRMsgT6sDHVtn4B", "proof_sha256": "7e40e4c6da7fd6e0801b7127a624b9f53a62cb846f8f2784d46b2541c9fa8c77" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Ionstream", "run_id": "20261025T172733Z", "generation_id": "gen-1792949332-laQbdofVW8SjyPWJMycu", "proof_sha256": "a7d9c43564064a8408dc81a3631d42ee5a540b48f1851c0a87e2f7f8d276fba5" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949357-7BkE8quPkwqR1dv3fP4s", "proof_sha256": "320eea5bb144871338094ca4d302ade243676c06d4f5463f57e718526475bf82" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792949393-rIjM80UgAf9ZtQ8S2G5Z", "proof_sha256": "25fbd21344426ea398f74fa1ae8f95af1bef5dda1ce536535b9998b2d6781723" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792949405-YZNjZeLktBOEgbVlkHW5", "proof_sha256": "7e5d791302a3f9fba3c626ec5396e707ab38426de60bd6655d6006d6375f5505" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "PrimeIntellect", "run_id": "20261025T172733Z", "generation_id": "gen-1792949415-6v7xxuSKVVITvU05L1Le", "proof_sha256": "8cceb5ff5ecf00eb0070c13393685f55aae0dad6c0df12abbd991415c050827f" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Fireworks", "run_id": "20261025T172733Z", "generation_id": "gen-1792949425-K2d6pqLiXvCnZ1JR8YZh", "proof_sha256": "f3854bbf7c0bc3550bbeb2b6ae5abe80b78ca95729f642866f69e47d7de4f489" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Morph", "run_id": "20261025T172733Z", "generation_id": "gen-1792949447-cyesumq2srpcOBVW1XCP", "proof_sha256": "cc0c36a69be8d0a550c539bbf7791bde5f9573e4d117089937953aa741175043" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Ionstream", "run_id": "20261025T172733Z", "generation_id": "gen-1792949486-NkFPYJ5VofRVW4F5pbUd", "proof_sha256": "11a9d7e1998d0c43be98ec62b826b09fbf27faae7f48ae3b3f2f9ff8b39d69d5" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949509-n3RJjPkx0iYCoKu1iYpc", "proof_sha256": "86c37ed0190227e13c64aa4e245583ffaf8ae41b17c97ad1802b7f4bf82a0ad6" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T172733Z", "generation_id": "gen-1792949539-lxFptWsVCVMDCSoUUQ6d", "proof_sha256": "3442a035cbefaa4001d98dce48e9d209a4f9ec8309ae7c2e35a063df1e5ff8a2" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Morph", "run_id": "20261025T172733Z", "generation_id": "gen-1792949557-RPSBCugznJf5SpFplWat", "proof_sha256": "84a70a789333324e0bf4e568bddc07da70bc0334b7b6007ae80d4c3d338e69bf" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Reka", "run_id": "20261025T172733Z", "generation_id": "gen-1792949566-Ck64vVOb4ecuCrNnrpS0", "proof_sha256": "d856dcf460e5093f66549ef524ff0c476a33b80cfaf3505e41bda530bb597b6f" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261025T172733Z", "generation_id": "gen-1792949587-lWW6OxV82YfjHSQuRIVM", "proof_sha256": "e1bb2a68b76c6bd8e968bf77bf6b17c0bedaddf4f40f775182f2923ba6738927" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Fireworks", "run_id": "20261025T172733Z", "generation_id": "gen-1792949595-g73dd6Rkek1ZSMzQKVyv", "proof_sha256": "151cc5959e958d3c7e3409532b9fde81280bf219381da22bd3e4e243a3f7be5a" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949605-RhIRgm1xCi2I5v8OaSNl", "proof_sha256": "f08dae3085bc357927d8d4a9a57e1ecb87ebcd92e6c7cde1a9855530d1d63b70" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AtlasCloud", "run_id": "20261025T172733Z", "generation_id": "gen-1792949614-6DVkxRiuZIJTY0ONRLsn", "proof_sha256": "5fada52a44a1b32d5aaf2af23ec5e2dad55eb323fcecf5ac5cf6db82618ea60b" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261025T172733Z", "generation_id": "gen-1792949630-ZJwHSLZw09SlkaQCNBhf", "proof_sha256": "f03f54f7b68c23aff2320f2414354461ebdb539e109e5c3c6c6f5edd96a44ec6" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Ionstream", "run_id": "20261025T172733Z", "generation_id": "gen-1792949638-4inbY13kyItIXkIZNu1x", "proof_sha256": "d15b60a41cc54c01e4f1a8582ce570e46c5cdaef01bb4ecabba903b0daae03f9" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Reka", "run_id": "20261025T172733Z", "generation_id": "gen-1792949648-oDxwe2m2eosDn7LoQ13a", "proof_sha256": "64837b1cd9504208284aa733354461169cacb7f5fd82298ba9018aeba8075303" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Fireworks", "run_id": "20261025T172733Z", "generation_id": "gen-1792949658-rpRlXWsw2d78iugqEan8", "proof_sha256": "458b38d47c48bd42987ec7c538a32c3cac6043ffef5d36aa23bd2ae2d7f1b2e9" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T172733Z", "generation_id": "gen-1792949680-oNPGi7MY8l8kVYnlAAvq", "proof_sha256": "2aa32917b37960f02d9080385f3dac0352c28a604ef7e6711f16fe6786893436" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792949694-0UUEXJTKcmdPhdNCUIqN", "proof_sha256": "fb259543071d02c9af01451ea3454fe00d54942449a15a31e0b7b106dd7d49ea" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Morph", "run_id": "20261025T172733Z", "generation_id": "gen-1792949723-TZ0GWlPbOpZYKskWmcy9", "proof_sha256": "7285113c16e1a10a7a2ffa8bdd9e357c3bbb1037582d6d4e144c158754ad0b3d" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261025T172733Z", "generation_id": "gen-1792949742-xiIseviu1Or2EdHR18iK", "proof_sha256": "27b7c72ba781d4045e5f37418ca5ab0ff237ecbf65d99d6d115bb61eef66f61b" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "InferenceNet", "run_id": "20261025T172733Z", "generation_id": "gen-1792949755-C3JZidhEfRgTLyBQUYWG", "proof_sha256": "fb39b1b437a2c1b95b194cfcf6504d38186302128577d0ac3f7640f9aa3bae30" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "AtlasCloud", "run_id": "20261025T172733Z", "generation_id": "gen-1792949790-LGd7b8pXSnxZQabgt7EM", "proof_sha256": "76365d40b075b3c514679256b0d8907780cd52a5a9556adc5574a76e780e89e3" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Fireworks", "run_id": "20261025T172733Z", "generation_id": "gen-1792949839-oR0pVnxXaMR6tddROBLG", "proof_sha256": "f29427c4ee1261f6767c9bd4d1605c6dc0f350737f1c4aa32b271d485d3a8895" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Relace", "run_id": "20261025T172733Z", "generation_id": "gen-1792949860-h41tPGfABKucQ9ZESE3K", "proof_sha256": "cb509b4dc1eccc578c2e164eb469de39c90e9a713831b25dac4338a3102d85e4" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Ionstream", "run_id": "20261025T172733Z", "generation_id": "gen-1792949894-4YFTXKD73NDQqTbs3hLP", "proof_sha256": "f2687846a84a6d0d1dbbc8c6375d4cf6876990b49a7572f47e65e51ff779b91a" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792949938-y7SoBoqRU5ayMBOYZeLR", "proof_sha256": "b395bcc04dac9403cdbdfef609ab675cee747f958e234c4d687b1f22f2a63809" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Relace", "run_id": "20261025T172733Z", "generation_id": "gen-1792949980-vTHI8b00EGluzTGdssIN", "proof_sha256": "31381bd211d2e4e74016ae6e9af02e5a39e1632dec4b43ababc6a09f16197066" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792950014-OgexbxHJepKxCIKZBUMs", "proof_sha256": "abac1d01756b004c6996c0bf9588112ef05ae07762699ad332816c31a5991639" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261025T172733Z", "generation_id": "gen-1792950025-eCjfJN6uOTFOIVnhjZ4K", "proof_sha256": "9e3a0c03e786a5ad74d30e7f5a268282dbcceae670063aa8948a251af2170d5c" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Cloudflare", "run_id": "20261025T172733Z", "generation_id": "gen-1792950045-hMn3tgKAtTUgQ8IAzgpx", "proof_sha256": "3b94edf5484f8c2c59b92ad14b70bc3d4a4dd6eec47b304b82c0a1d7529ddc6e" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792950086-u6E9t8OXnHMAkJFd8xXx", "proof_sha256": "4f1d7c23c27f19c4d6df9a32e16819c1ec1f186af30c4519a7ef24b086fc2088" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792950116-oUl7DJkQrDu8k8tnM9TR", "proof_sha256": "c86cf122b948a5c76936bf04bccfc8ba16e7cd4b2e5d1790aee9b50f2a4d036e" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Decart", "run_id": "20261025T172733Z", "generation_id": "gen-1792950162-wO4gk2CofwQXdOlbYuQe", "proof_sha256": "be18f6553247482368e9a715f5e746394fa49efd29603eaf4f44feb848505583" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792950190-VyOwHcGYOOXflI2VT10B", "proof_sha256": "74e635ebd446b0f839a1414636a04e5832aac9724f86f1ceca030899a283841b" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Morph", "run_id": "20261025T172733Z", "generation_id": "gen-1792950232-3oex01meh6m1JVZhil8i", "proof_sha256": "6cd9cd21c4013468cd9667c78b477e2d195de159c1f8b7642a6670677a69eebe" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Sail Research", "run_id": "20261025T172733Z", "generation_id": "gen-1792950251-ZNJSz2G0VEixSvtxBh7t", "proof_sha256": "720d07c87a438ee872c8962e44a74956ad5b35ff9174376d93364c1d92ff6c8f" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Decart", "run_id": "20261025T172733Z", "generation_id": "gen-1792950324-GWUl3j7yKDPHUjAFgAIE", "proof_sha256": "126b379deceae3f72ad899f2e76c7ec678f14a8843f86e6332ad5bf6d86faf67" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Fireworks", "run_id": "20261025T172733Z", "generation_id": "gen-1792950384-YfHyvFeFNtOqoZvPE1W5", "proof_sha256": "22f3cc0d72fa179a02e5cf3f1644d95fb7c680931866803f56ae475bbd243b9c" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792950402-kGZYhcewyzgWL1dPVOnf", "proof_sha256": "514e8b2642f5672dd472ffd6bbd98d17ba5865c1247b0c7c140907643914d9f9" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261025T172733Z", "generation_id": "gen-1792950421-6p4Z7xjciGJWTI8Todrn", "proof_sha256": "ad00b4d9bb50a9b1a52e9738f13141abce3da678ee5044732a2fe5dc973205bd" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792950430-UEebL4jrWqD55dbKXaXS", "proof_sha256": "e1cbe3ca6c6bca4d999520f41bce61065400a2fa7d1c3d791fac46b237a94710" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792950448-w2vjvpxO2JlIaiwlGLiq", "proof_sha256": "d4371115e3afa5e8d877ea17f1bd6810db5a28645527f0d2ae3f6f56e0391dfd" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Ionstream", "run_id": "20261025T172733Z", "generation_id": "gen-1792950461-ViubohlhisqgQY0mHNrZ", "proof_sha256": "bb1fd9efc8b44f0de7af9b611936507111d511a6f533761712ffabe6e3b2561d" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Ionstream", "run_id": "20261025T172733Z", "generation_id": "gen-1792950480-DNPlMFVEiichuk5Dws4b", "proof_sha256": "47f15617be2c359bb21aff3d1a30b2537136e14eb3550b2207dd870c9a47fba3" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261025T172733Z", "generation_id": "gen-1792950514-t6gFil7oxJdvgj6WsWP2", "proof_sha256": "527751d1e6913dbdb38531c94cbeacc8ac098cb69bc92b0f7910d1a8a56b4ccf" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Fireworks", "run_id": "20261025T172733Z", "generation_id": "gen-1792950526-KsYTXM5C4HVrwoogGAb2", "proof_sha256": "3a7e25e3f7babab0696acd5d2baa2bbdf13a6bc83031e8af72f2dc6c031a394a" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261025T172733Z", "generation_id": "gen-1792950536-NKB7nzGBH5KLy7Oa083B", "proof_sha256": "08ce0768e443175928c1322eaccc5c8a3cfcd80d62fc6ea1860a528ee13179a1" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T172733Z", "generation_id": "gen-1792950561-qSbIV31K8gXzK80qFw7r", "proof_sha256": "b177eaf8d54a975108faab9115e5105e8e11c43b1879d4c8857615b4fee3663f" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261025T172733Z", "generation_id": "gen-1792950603-KMo7yOWmlGPCKlQRiwBS", "proof_sha256": "dcc8d4fd2b7e48c4341782e258417acd9f3806199488d4e77f07477bae6a0903" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792950613-8EXYky4ubuT1EmvTgeB4", "proof_sha256": "deae6ef4e45335cbdaa3c828fc0b50fef975684fea5af3bd5d73b3add966de1f" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T172733Z", "generation_id": "gen-1792950623-OoTB40cmO9hXee2ZZDaU", "proof_sha256": "81332d852a1c4494e0465842997228d2736e2aca1224bb4ca5869931441dd72a" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Morph", "run_id": "20261025T172733Z", "generation_id": "gen-1792950631-ekNOJE3Oz4bjdVJhsge8", "proof_sha256": "6b290b9c32caf6a137a926462f4d848dee70dc110ef8e771786c7ced124fba11" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792950640-jkKIT1YYb9pcMvWmyBp8", "proof_sha256": "33f815b11a86d9f8a2251568a1ed3eb908c044bccc9889e80f42fd1e37a44f85" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261025T172733Z", "generation_id": "gen-1792950648-jCUouwMlvJJaMGIGNF0X", "proof_sha256": "81bc83d2e4eaff50bbe5c3de0288bfa8140ed6b80cf4c30e4b1b135b6b6b59c5" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Reka", "run_id": "20261025T172733Z", "generation_id": "gen-1792950655-5HemBm7uFX2cReQPzj7V", "proof_sha256": "40556a7139279800e2c1daa6d261cbb168b637a59f8bbe5245b8bcce7402cb72" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Makora", "run_id": "20261025T172733Z", "generation_id": "gen-1792950663-LAjVa1VDGQd0iYMKnTLk", "proof_sha256": "8ba82ad6b424437e8dc281b7935560eb46f4d4875a53911c806e164d8cbde9a2" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Reka", "run_id": "20261025T172733Z", "generation_id": "gen-1792950673-SCJSqJwDgRwPKPopqDov", "proof_sha256": "3946e19b2478cca4701deaf098f2abb1269ae2cadc442a9a4751806f19ffadee" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Morph", "run_id": "20261025T172733Z", "generation_id": "gen-1792950681-MSogmSWPZckSjUcfBk07", "proof_sha256": "d2817c18c2fe237acf5210cf139e1b711bf6ed3e7f47f853a22ee977dfe18a48" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261025T172733Z", "generation_id": "gen-1792950691-Hh75zOwjNtc5PG5pWy0o", "proof_sha256": "bda38c8860ebc3427986bec390202a22081ea41103b1fbe9e3cc2438264be5fb" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792950699-jJDX0R3LSStLCQR3UJVc", "proof_sha256": "f36517d9090eb77aa3ff22f0c1ea7b60bb75ddbf0fd1522032bf1c5b89632d41" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261025T172733Z", "generation_id": "gen-1792950707-3c5341sFYndXSGLvXyHT", "proof_sha256": "e8f495167823cb9348bfb059295698fe70f639b8cfaa39ae6196888b73f4d8cc" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Fireworks", "run_id": "20261025T172733Z", "generation_id": "gen-1792950735-TNbbZKWUh2qIDAAVCVXB", "proof_sha256": "eb060b01a93e36f7c43f4b7bcadc12964703f11e6ba0fb0341bc99d7dff98bcc" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Ionstream", "run_id": "20261025T172733Z", "generation_id": "gen-1792950743-zCUhyyTxtNK5SBVmgVk3", "proof_sha256": "6c0cbc0f90f8e3bdfd21cdb27a535972c01c9171282c3a2149b51e7a7d5ee558" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T172733Z", "generation_id": "gen-1792950751-ORWVvfLccwbxUkcUEVc6", "proof_sha256": "1ce52d74523c98ca24f7e3d71665e870986969c92b98f25f87aa279d47df3820" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792950760-BYSNYf4wll2vdyhqUb9C", "proof_sha256": "89e2ac3d871c561ecc2126facd1b7ac2f65868a3730cb4fe835fd775fb21c6a3" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Decart", "run_id": "20261025T172733Z", "generation_id": "gen-1792950785-fzewXyJ1LCrWl8uIFSDv", "proof_sha256": "f55b10a6e3d0fe86f4981920a8feb7487a0135385f847d1b40472223d4209fa6" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261025T172733Z", "generation_id": "gen-1792950794-L5T4hGQve5LfMPHztTq7", "proof_sha256": "2005a200f4094326809ba0b471d2246d4cb3863ccf1c64d0ccaf32e0919f7f12" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Morph", "run_id": "20261025T172733Z", "generation_id": "gen-1792950803-M7CVHQg8a4cEApv5H5r9", "proof_sha256": "54f9a0619f78d6464905e6809ee7e0ae129187d624ec3dc0c2296962a75d6c35" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792950820-bEOVokBiUgZ5VPFUPSky", "proof_sha256": "28956e1c35c64f54c86a9ea46490933f950311496d362d737358b1f7cfc9278c" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792950828-AdVQigwyDIQsAeKgMbwu", "proof_sha256": "224b28a447179f11a072064db4a798d70f5086be0b8cfa655068b20eb46ee930" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792950848-I8RedYKY2rbSnT0xxHqW", "proof_sha256": "0150c500ad2b662072e84c48d706b70e5133e0c69b598b7f1017990591f66813" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792950856-WriSRsezUJLgC8E74QAw", "proof_sha256": "173e3ee31c0c45fd0324eff066ab71888589438e5f7a41c89082e2b20cd58d43" }, { "model": "deepseek/deepseek-v4.1-flash-1017", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Fireworks", "run_id": "20261025T172733Z", "generation_id": "gen-1792950866-pLSD288gLHNyv4QAUZS2", "proof_sha256": "232a7647fa0868810c53690188bbde3928929742a5e900fa7e1e6117b6ca65a1" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949253-aiw9j5u2iEEHIqqQhpqi", "proof_sha256": "f1c375f43c6b5d56ac9578577e1b4b3734ccbf7ec7fa14d0ee9a6e58af4fc6c3" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "c and e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949277-RbCO1BcqdOe7AmEZloea", "proof_sha256": "16a9bcc40c8453c29135c01785ebebc51cc699f137df41443d4aca7cdd090cbc" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949289-Lhr3lsy9UfV38I3bowle", "proof_sha256": "06130581ff022ce4a4bcca97305aeec1c15fda38beefa7050a01691480b537b2" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949299-x45NS3ozRYRN3T3oLjIj", "proof_sha256": "0de301bba2e5bf61350a81a9e917e9c57e2a1ffc831e4a2e0b332641db85bc1d" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949314-VNyLg2YvwAE1Creomjoj", "proof_sha256": "19378a6aadae92f087333f475a983bca0f5150129abd354ee2a1b3f6e4a2d336" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949338-kZK9n8phoMWgXueSInCe", "proof_sha256": "d6908e546869bb3f63e8917547777ae63b1b5b7c31f614949ce4647ecbd514f4" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949351-dLSYdUuyhLygEahY54sg", "proof_sha256": "287bd3643b4e0974783403adab01d1f6e6da859777a731254297230c7317fd15" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949362-3fdlTOelJ8Vzisijq2dn", "proof_sha256": "a3110e34cec9ab701dc960bbf1beccd24c41701b39566ed02dc92d7b2cc4cb36" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949384-jJbBjVJfBeg326cqJxLa", "proof_sha256": "ba146f2b65e05c35ba1de488c652c0bcf130abf698b8375944a3c7eafba668e0" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949403-cMF3HsIFks8gD6ivEdph", "proof_sha256": "151d267e715074b8fc4a822531fcc70579afc2f78b45c5afd4cc808c96de144e" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949429-6EoHbEDYG406qDa1NxES", "proof_sha256": "0d621480680b542102c08b9b0fabdf923f0a608d636bb360756af600ac5ff168" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949447-7Cj8xRZyyyo08ZH5n6EK", "proof_sha256": "a682300c265406e1a386f76778ba7a3dd5822a52a5738f63d48a6574ad79b583" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949485-DTl17b0q3O0baKmAFAQE", "proof_sha256": "6a91b72de72c0534faea650d77acf2bcf719b46e4259e573ec2813bdda813494" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949504-pBBmOvfhVWB1g0Rwzdd7", "proof_sha256": "df51a9339d151a064e42abc138b4c16bddabbc70d3f267a31637155c6afc4651" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949527-uhBO158jWs3lxElggZDX", "proof_sha256": "259a6087c12fad7ebf295e377b89a8066d434ca11422c98f6cbc4254f3535769" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949542-XRAL3YPy5PWLdZNZSxmI", "proof_sha256": "67e9091715ab141d2959dc1dc5695c0feb1109d4aa784402c273a97def8dd62e" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949553-jtD1ofYBcfsiTig4jlPn", "proof_sha256": "2ed9347e16992ce260320a8db61aeb1a3b4b5f62b9f7cfec378f6407017e720a" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949563-sSjbKPlSoEZQAJOYWeoK", "proof_sha256": "15a0e548ed6b60c374a6452ffb02cc90de25c823be5262a1a2980c167337828a" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949585-iImb8g1aolyVk4UYUPW3", "proof_sha256": "0c6f927893291f7074f5a530570176700bf90d1eaaae03aae0e0894c55eb8bc1" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949597-qyI8ClJ2X4wZA2IKpidY", "proof_sha256": "5a62a75e078e086429afbbc3df470530566f5c762fb3d422a47199dfc4d61b30" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949616-vOo0J6oFpvSBYfemmkse", "proof_sha256": "3ad96ecf423c459648499df6c99bd3fec4351c2355cef6b76fd2e658a0b441a7" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949625-Jh1sKzPAgFDiA5uE1eJg", "proof_sha256": "aaff1d63e0b3d88e29525062cf764e1f45f6e003ba2885eb56f259b10506397f" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949638-tYzkzY5A6aUTUPZQvQcf", "proof_sha256": "e35ae6b126363a2d3c31de866434ee6417f654bd9c2317a4f8b4a6dc81a23b75" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949697-XkqE6Xl8naHlxg98Q4JC", "proof_sha256": "126ffbfb630b751dce09d09aae6ffb160c194c7c39f53ebdb65df1dd81305baf" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949717-4rWWUVE58ejOnowLFR4F", "proof_sha256": "035ef055f098f9a59d2c123a409c53fce17b558f01cb28a17254ce63ab8870d0" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949745-GDrtAqoe4C6lnfJi8qDU", "proof_sha256": "3765887ca0035c3007a32d01444a7cb9b603161e06da0302f083937b8aebf194" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949768-ypgtdPJCkGuerRjgp0W5", "proof_sha256": "b339dbc0986747cf22145e774160f399d85ccb99a4a12d68855d2a44679fd83c" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949780-7ycf9qlUAUNTb842lSkr", "proof_sha256": "dc186aa8cd1972b01bbd74867f04299b1d11683d7a0def2e861e47b8da7bf0f7" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949804-xOvPEZ4l4YP0x76IXblV", "proof_sha256": "b041320589d1fb6920fc66c9feb4b1cc89d7c3f11836fa67ca28145506d4413b" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949826-4aKF5sIn2Ns9jVHluqXj", "proof_sha256": "bb0156e273cda4eed1f587d7ce3daecd0bcad0ad7c3b1f1f3a1bc9d6c3684503" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949838-tMfRuwBCKCi4cJzCBCGy", "proof_sha256": "1567a0e06c76abeda53f88cfcd04497c295c300e3c86d068fb6dbc45ce860795" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949859-pdmVEjRhVOsEWHgc2f3T", "proof_sha256": "7fb399a448a83cb3cc2cc5b3483f8a7c4cbd0d23b2fdf5f74ba79bc5919f4386" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949880-kKDSBTLjSlyJYRq4JJ2q", "proof_sha256": "937fd2d633ca7232bc046c80b7378e382ad7854611d0ad9516ee4de6b2dc43ec" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949892-EeYwhkclVRzc6s25SbEq", "proof_sha256": "3325643f7b029d6cd710e91f9c78ea5cc17d6ac76dfab16ebe0e8cf72004ff02" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949916-vLFTSd06TWWWDWGNTtSL", "proof_sha256": "eafb30f399e9491fc88ef418b53af4143bc007613d3c1c0710f83deb4a3582d7" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949949-WneiKyMFiMpUpZWy3SPf", "proof_sha256": "6ff2816252d128723ac2b44614634f89b6b2703e174729ab633779d25b106244" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949992-H33wAQl9dGEdck0Jm6bC", "proof_sha256": "44ac6a9b56a0dd22df33e0bd2d6887b7d9ef4644325f0dacc7728f130a0eada6" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950017-QrzXf7LM0uhUl5DwiUu8", "proof_sha256": "16da0dd46bbfdf2c84795419639c5742eebd337bce3147b476aa320b540534eb" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950043-HJi3dSpdPhF6Usj8MHLS", "proof_sha256": "1a9163c0b3069bf0fd562b065d393db5ade84b5afb50cd93304a046130d1e313" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950070-y7BQEBsPiX24KsjU7UAA", "proof_sha256": "d7027b74591964f7806ccda3be59651c0bf0ad854145a4dfb6908b9040ee9c89" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950089-Z6fCIfFg2XiBxYBOuiQV", "proof_sha256": "9576f1c4aad210ce85db7dfbb60a1d75524dfade89af02f16772101f26f1366e" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950098-5cPFWmXwxUQpMTrnmW49", "proof_sha256": "5b6b9a2e2e552352fc36a2aaa6c66b948c3585c7e75cf23d599110639d2c0eb6" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950118-Vkhoh77z9dx0GkhDJlWf", "proof_sha256": "426c017696a6913928c02f65c3d31588fc78143615a71a449130c4e064837b32" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950160-H2jmQ2CvfkN1xDavaBmv", "proof_sha256": "ebba0ff96e345fdd53ab171b7e79288ba6aaab7f3e6e1d534b5a125fbb760304" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950212-PRPAuUG8XSZGjggPg6UB", "proof_sha256": "8ddcefd509be034f85c6ea84416a3b54d1b6360a12a4f969ab636d7037a56e16" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950244-ZRfPPA0R5BBTphguO2wt", "proof_sha256": "58a7d1cdf416139354fd0cd80d64fe372b116aa7f947e04c411bd5e4fc5649b5" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950265-BJpSMEMIFQFXYhXcLHA7", "proof_sha256": "6270e97db08debc5f511d77f057782c299b46b3a346ef2cc61de0f6ac913fda4" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950284-xUjK7YKcNDm3qFTiVhU6", "proof_sha256": "8d63b44124db9b3e9d1c93c6f4d157a5c5da07035eb522ce137822029710e603" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950323-5zsf6o4ogc7f47ZQCEYG", "proof_sha256": "c2923075585afa9b24d217a13fadfaae43cef60537fc64a1fc301a93f9e200f0" }, { "model": "openai/gpt-6.1-sol", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950365-X6nlKpZh8CN3VSLdJ81r", "proof_sha256": "54a2d0679d0c5b5f28945ff26ad73af775e513339191be31399da3772e5fe8d6" }, { "model": "openai/gpt-6.1-sol", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950390-UU1h9EggnmoSEIzE7TO8", "proof_sha256": "f02ad9fa749810032e612f38b01d0e6803e3eac65c98feeedfb63f19b5902fea" }, { "model": "openai/gpt-6.1-sol", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950402-aUMsR8l9QFbSg43G2w1q", "proof_sha256": "c1e6b37230a90958f49f2dd81de7002f08109e412ff130b42f3bd4dd26f30da6" }, { "model": "openai/gpt-6.1-sol", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950413-OuP8u25xDED9OQX9Huw2", "proof_sha256": "44ff36ed86d006ead5c0a2938e32ac740769122f0feec9e03d91f148bb6a3dd3" }, { "model": "openai/gpt-6.1-sol", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950424-3jDlMnZTEf5ELps8X6Wu", "proof_sha256": "dddae743a6ec5e0d007c2d9e5c5d6c87577f10ada4d1c60bdd2b9c728bcf29b1" }, { "model": "openai/gpt-6.1-sol", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950435-QcrLStM0dTSKhBA3hA7R", "proof_sha256": "d32d92af7622caeb34b143af6a68da416a5d902309524f830b40cb1f16ff93e4" }, { "model": "openai/gpt-6.1-sol", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950447-0ijmsPMb6JfJXbwB2I2g", "proof_sha256": "1e1be3726fa3490b86eb89a493ef4610e23ded0080e1d9d385528a22f0b2a767" }, { "model": "openai/gpt-6.1-sol", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950456-8Tacypq0yXy2Rk1mCvdX", "proof_sha256": "c2b7b4b8fb489bc81de2697a0b767a31ad77516468d6bb3d885d7227cdaae4b0" }, { "model": "openai/gpt-6.1-sol", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950465-27x1l0zQJB7dsSNnexn3", "proof_sha256": "c715a9bd09e6568b53281496f6332234b7562b8a96f94d3fb1f9950ba32d6eb1" }, { "model": "openai/gpt-6.1-sol", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950475-IOcOQ5dRyyaWIAHKyYR0", "proof_sha256": "456b3f1c20958928c4fbb15362a5c637e6cc92dc354a3f86e13a63ed95708cdd" }, { "model": "openai/gpt-6.1-sol", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950484-SzPGTl3vOUE77TFfTdPE", "proof_sha256": "d6351579673f565814867ed1405d8e5c77bee6d999fa682ced5adad00792c35c" }, { "model": "openai/gpt-6.1-sol", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950495-gh6Nu8Du5mUQo1HXavrw", "proof_sha256": "6e1d030ce011336d8ee4f46831b3227a73b32ae9dba2baddf0c8646f1a56e93d" }, { "model": "openai/gpt-6.1-sol", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950512-mJXT1OpOd8A6SCjL7vG6", "proof_sha256": "6790a156ef0cd2f1361eb0f80ed9de3801fb94ad8ac5348dfeb15c29b7ac5490" }, { "model": "openai/gpt-6.1-sol", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950520-OVAvq4JedXHerw2xoEys", "proof_sha256": "0ce1bb1fa12ffe6782f1c4d56465d8c09c39a071fd5734f4e5f56a51a321f861" }, { "model": "openai/gpt-6.1-sol", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950529-zwc5Jek39GahaivFDGPV", "proof_sha256": "bf0c243fe9cb5d06d63ac3ae3fea61a826900e8a4b0f6596fdb3e8db00e716f5" }, { "model": "openai/gpt-6.1-sol", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950539-P1LeuffOvcutmMsHZs2U", "proof_sha256": "dcb42861448cc54008d63c50442bcbb7f832b8a6c6fddd63fdaee98e641dc33f" }, { "model": "openai/gpt-6.1-sol", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950547-flXLMmZjLP9AxTWXR431", "proof_sha256": "5356e1be5fd24edef8f853e87e7efce555d740ebc66773ea3e959204ceb185e3" }, { "model": "openai/gpt-6.1-sol", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950557-W32BizFGKqpMmsmLeH11", "proof_sha256": "0a26eccd4bd92d8948f03a3c7eb259288d81dbba60b66660e49faeee8d561911" }, { "model": "openai/gpt-6.1-sol", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950576-ioPV22UKYUWl67Q5zZcS", "proof_sha256": "60fbcfa8a97c0f4d1f5b83a246336ed98ebd746b033e774824151129da92054d" }, { "model": "openai/gpt-6.1-sol", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950586-gnnPFtyI8ltu2Gyznqdh", "proof_sha256": "16da576851862088d4298aca94bd5e211f1155edf4201456d8db3de8873643ff" }, { "model": "openai/gpt-6.1-sol", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950604-FKfDdpxRAzO9AY3RRoMZ", "proof_sha256": "a24058a46622664df76fed50bfcb568e473a8df1e46d406ce859ac18d2e87057" }, { "model": "openai/gpt-6.1-sol", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950613-0Yhp27qUfd5OkoY8Vuk1", "proof_sha256": "a18f117a8b9fb1ee3a39471ca87aefff3992aa6f23c03519224c18c2b44a4477" }, { "model": "openai/gpt-6.1-sol", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950621-A6tYVEiN73th69PS6Mub", "proof_sha256": "188c5e6c4011cb80a50770bd3a17910180341a3aa1a9b5e253fdb68ba13aa0d9" }, { "model": "openai/gpt-6.1-sol", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950633-U3WOUtNVydQTxsfqMsMg", "proof_sha256": "4fea66ef41c5ac30f53ca26715a2298911ae6f16e85598fe9586e32600974e55" }, { "model": "openai/gpt-6.1-sol", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950642-o8JWzxVvHgsrsybV19jV", "proof_sha256": "23135217188f9948da86958d06ec685c37ffecd1f3ce8cd7c08aa0c1e08a21fa" }, { "model": "openai/gpt-6.1-sol", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950661-xFPi9rQV0SbEDxzK1YiD", "proof_sha256": "077b8ddb845a8feb2d2dbf6a4e9e656a6c090336b541d31c12c410d1f852edbc" }, { "model": "openai/gpt-6.1-sol", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950670-JQ0d1DvDv4Kvr5sdlETm", "proof_sha256": "7a4553a2de8c4e1ec9fc48cb2478dd29fdd2c590aea5a3b08a2a319dbc2fb7ad" }, { "model": "openai/gpt-6.1-sol", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950680-mPKzBGfpMuhaZ18082aw", "proof_sha256": "240cb87a981141ed85cedb354b6edfc87e500dc97e3a16e8aeffbab2d6f233ba" }, { "model": "openai/gpt-6.1-sol", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950692-8TAHqDozndEqJOXWfzuo", "proof_sha256": "887c3c404891693638cf8f4a736c905cadac881fb4cfbc9abaa8c091b3e9de3f" }, { "model": "openai/gpt-6.1-sol", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950701-Y033W7igIpMzY3ImY3ze", "proof_sha256": "698a430494ea1206b68db63f7a52e297166734044c949474be74b3ac2e1f38c6" }, { "model": "openai/gpt-6.1-sol", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950710-ngDoTKoXDyOimEweX5HN", "proof_sha256": "b18691df394ce42ced3e95754327b15705428f16d717cd1a98734560740dad3d" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949253-KM7PnS4e1HmvbRdJJV29", "proof_sha256": "9e987860cae118201659fb0aa92e4e239529126a3fff1d2815b7fc71196390e7" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949280-MPmA7lalDMcOhVUJZolK", "proof_sha256": "481b6fbd12342ebac80268b73e8cdc731d12d190e1ec50280dc4fd53b4753450" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949299-3OtbdQRp5r86jIhLFrgj", "proof_sha256": "1a298c231296d0da9459c16cb89ff5b17219f75c113aa8f50a8c5c815cefe402" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949316-NJnKCN5JyUAGC8O7oT4c", "proof_sha256": "d625cfc2a5b0971cf8403416cce04dbfbd48c1fd2454793cbb208b1bb82c52a2" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949339-ipjDFQddqRNoRWCAicjy", "proof_sha256": "2454284de62573358941e6b10e0fbaab6978da4eb54b748f2a3251d0601d86e4" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949384-gUvaCZieZL9VfmlKAfQt", "proof_sha256": "8f54828cfa2b75b59b16ab68a575151be59bedc648b4b4325f7db38a3c6ab78d" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949407-7vycCWN6FV4GzqhciyVI", "proof_sha256": "4a6b82742686939f0c5763556264c3765bec1dc8013e635bc18231648fe00abb" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949425-vDbG0XkK8dHjpmnOn2q5", "proof_sha256": "60bf37d446305344e9eafbf097acdb684f9d50845d4a057a5495d2aaf636f32c" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949451-GHQFDLde8moTFGwE3yeJ", "proof_sha256": "9645c66bc14159c30b1c335012c60c61183553b66b0be3703f4f2e7793bd1eb2" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949479-uUSme6ZcG8rhidR41Mu9", "proof_sha256": "195e7b765505e876b62f24171b8097d8b8d6b072c597f609872e29863cfaba71" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949511-1AgpeSWAjkC4eN6XpgGk", "proof_sha256": "a869499128fae8136402a0d512368180f44a8700e51da91014a672b11ba95ec7" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949532-LM7K70BJNZnlggC8bUIA", "proof_sha256": "1d69f951492c76661240562ca8359c781fa965f3b920b48b50a231c66ce50037" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949553-sgkLpHQ0I0eJNnCMXA1x", "proof_sha256": "679457f5be17a9c16c8663b06afa2359e2d2ba3adbe9e8c133632013db887b0b" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949577-SQrvZrfNUwj6bv4Avian", "proof_sha256": "a461b55b5def51f97114b02827d64f9160a2b13f9420f63d850038d0df0376f6" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949603-On1vU71YKtIwBC7ELZKn", "proof_sha256": "19e290295afcff61ad08d755135111a2377850daf97c0a4731a19e57ddcf3171" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949625-636HftkMP2a7t2x4A69s", "proof_sha256": "728c7b69d3a35c37ea71a39d1ff4e8e430e052906341a993994508d7ef0c63fb" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949643-Zvlp6eq9Ex3MNF1ik20i", "proof_sha256": "ee5adc0fb8d810d310482bd01b2a37c4ba88b69e73ded34516b0bd0b891e7377" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949659-C0RLnO4NtTBlTRslbyNW", "proof_sha256": "3884f739a4a42ca5297081cd08f858cce2f5772a1fe3ca9f134520c11c042f96" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949682-kuxa9GpSWvoLATxh3Kg6", "proof_sha256": "1a0b597c3204f2af83749559b2b19ae8ea28daeea090ddb6bb79046bc83f1d37" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949702-LL62zPZ004FHhUzFOYXk", "proof_sha256": "d66732414d33c0312d0ff6aa0b01a675f208de65e2041730a9bb7097b723a5d6" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949716-KJ3jnD4QYqYVXwlaBadT", "proof_sha256": "e31d3e3d350da4eba176381f99f8b497bb35db35d607ce0fc818d3c5773deb87" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949745-D49DpgsB48BrrRGp2qgO", "proof_sha256": "18f367d7e6ed27a8e903cb0240d88808bb9d69ee092f6e5ae98e07f14a4914f4" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": null, "correct": false, "errored": false, "refused": true, "excluded_not_in_manifest": false, "finish_reason": null, "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792951013-RYo0plQ9hcEo2cZLunig", "proof_sha256": "abaab4818b33e41974fbacd07d3fdc8be2092b92aa591cd09d45ef216158004b" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949772-fEmcVVol88VOSSLoN1bM", "proof_sha256": "60bd16ba74a01a436b3151d71f54e146db82c523c6dff55a190d63865a380302" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949790-7yM80KwwxwQXzMmOXxRm", "proof_sha256": "b1963a6f09db28c4a6dab0b7007914044aa5d2af5c2d6b412a960bba435b03ab" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949809-OhnzTMTVG4womH4QPPVt", "proof_sha256": "c10a887e2a1731fc454e1760df4d223250fa74846e186e060158437a41256db8" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949822-sjpLyKyNziSHuTxeJ71J", "proof_sha256": "2d6c74a5a9dc6f605d2bc204d46f9e34b6c872c1f9054f24e7f20f85753b5ed5" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949842-bf5cZ3r6gdIHRbxohk6l", "proof_sha256": "36d1ed1c0f560f87296b0e21deef82bd1b9e2caae65264ef75a58967c2e245dc" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949876-BtMltqigEz0Xwp9F1sNJ", "proof_sha256": "e12f60a4fab073f65937818435b7c3882072fb455ab0f9c74a26c0646c19f833" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google AI Studio", "run_id": "20261025T172733Z", "generation_id": "gen-1792949897-H7vZHy1l5Mbi7O2LCSEY", "proof_sha256": "7b2d578e169db5bc8510b033eed814d19c297c18f58c4281d65e7c7f53e99b95" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949937-HCWpyTiaUerbweyn1z0A", "proof_sha256": "4633eb7a843e3181c613ca94a1716e13cda37f00152c070bd6961a2327d58b0d" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792949987-ZgCtmvCH47b2E2vvJYUk", "proof_sha256": "9fa99644fe85a57276dd4e2e0ee2329f02025acff10d901c2d45d53147d84bf4" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google AI Studio", "run_id": "20261025T172733Z", "generation_id": "gen-1792950021-j8oOTiSUbrrvK2p0lzjy", "proof_sha256": "f1c06cc7ea1bcb0240fbda015cd2580862bfb17d3a979ff31139172f05ab55d8" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950065-nt8YyuKHM2nlnnNurEbS", "proof_sha256": "090a0cc46abaaee9b71332f04662431765708b8488511f49030f5dc875463b7a" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google AI Studio", "run_id": "20261025T172733Z", "generation_id": "gen-1792950093-lkP4Vy3U49L2SypwnYxX", "proof_sha256": "d3132dafff54309ee7b5dde5f5e21613fa5f2b49719e3c0b7cd9bf9481667a26" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950149-6fciUYnTvwanfHjjVsv9", "proof_sha256": "9ffe15602f4e6138471f4d245c404cc8848f4c93c4654b954d3e8262f9c94dd9" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950190-ZXLVYTts8zxIpkLVx90T", "proof_sha256": "acb52c9c7948524f75c72bfd8a5e5d92bcb88bbada41c7c40700db49bc445962" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950238-QFszzPdvzRXE78FOpuLh", "proof_sha256": "948a6226660699e6996c407a052c201b1e848353d39d8f6f4b85cf55fd3f30b1" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950259-BKUSIeq5hupNgO0iXRqh", "proof_sha256": "5bcd1dc1d30752f91f490855ae221169853c7648df40a8f352cfe6eaf43f8c45" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950308-ZufB1RLP2bJf9HlHhMMI", "proof_sha256": "f91e5eb6ccce77c073b70d961340acc75bff2e196f23ff2de3b4f5218df8a7d8" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950344-kFJdaKC4Of839lVwH9f8", "proof_sha256": "72f6fabeea2b35723f825abef346685f292e16192275e4c059ef4f9455be239a" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950387-BKS5IigYBjvV5cz1WRl9", "proof_sha256": "969c9f83446df97e0dd1fb5ec5c92d4f2f84dd09f4bb59bf45d7154371dbeb61" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950401-xvx6n1VtpURf5jztAVeO", "proof_sha256": "080089e11af2600a0c33a42077c1d381f464d60570edf7bae525f616357fbb27" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950419-ad79Niz00YU1FnZxiWKc", "proof_sha256": "f31cf2a83b99076ebb8377c392b30e7357d58045cafa10bcb32b274ea77dcc73" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950441-nRQgmLz0JrXxdOs6COX4", "proof_sha256": "789957fd26e6cbbe6f5a53b90867cf534cb23d42bc5a584eddbab90f55a7cbaa" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950456-FgGBZyPcegPA5oAk0qED", "proof_sha256": "9ee0f1098938700ba58c2240c413487f86dcdc1bc9c9dc4170f06af5eeecb9ea" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950485-xYdYYrQgkoiWoOV4wAQB", "proof_sha256": "22f4509e922362f5e25011a80b5f02d730e61ae37a4878586e88556d48da5406" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950498-boGLwFlFvuXRvjdysJqj", "proof_sha256": "98aac7917ba584ca98b9b052ab08325a7e2cc3ab3cfcbdb4eb5b19309edd2049" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950519-2pwldj8G6lWvPTHtRvh5", "proof_sha256": "870ec148f498a4a1171f75c01efb061e98ca4c9a2be997ad38e6cefb78435b79" }, { "model": "google/gemini-4-argon", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950542-xxT5Fchb5rEcSQo6TBbI", "proof_sha256": "7b6b2090273359cfb562c9503177ea8fcc763fc75cfc99e539e80fdedaba2ff0" }, { "model": "google/gemini-4-argon", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950556-yK17WRHRLCoG5NGSyba9", "proof_sha256": "27206b716de4d1cd96402a5edadd4cd779a08dddcbab965e38484b12184a7993" }, { "model": "google/gemini-4-argon", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950575-gRY5i1qscBYtNUlOd3Y1", "proof_sha256": "31d7dbbace91bedce3dc174d7b0f45e6dc2cdc7b742873f74ef5351cd4de0d13" }, { "model": "google/gemini-4-argon", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950592-n0wY2iej0H8hfd0ZE2UH", "proof_sha256": "e30fc661f87f9a9a553bf20155a528e192a83818ebcd10326788d7b629b969fc" }, { "model": "google/gemini-4-argon", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950625-GuelAems6DhO07aqda1I", "proof_sha256": "3a89629a49ebaf7139e760116518b08d64e81495366ad41f73450801fee36296" }, { "model": "google/gemini-4-argon", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950646-OvBTpOEYRTdmARh1CTE0", "proof_sha256": "23f5762ef98e87798c97282c9189df622f4165f19004402e4d1206132c50ac71" }, { "model": "google/gemini-4-argon", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950668-SczwpQ0Y1eS6iOnwcdJw", "proof_sha256": "a3b2b865fef5c989074a109f307d72aa2c7aafaa25af238c725313ad749cd032" }, { "model": "google/gemini-4-argon", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google AI Studio", "run_id": "20261025T172733Z", "generation_id": "gen-1792950680-GHCh6jRll9pHOAr4jGwj", "proof_sha256": "4eb40797385d9214f49aa1989c79e1b213297c3e1a0e2123b3ee46bbcb94868f" }, { "model": "google/gemini-4-argon", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950701-piG03qX9PkSWnCmvluz9", "proof_sha256": "2a9fe6bac7e3161560fe5eb7ef2de3d911c02f98b36fa71e6231cde519090264" }, { "model": "google/gemini-4-argon", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950725-T110ecaYYa5obVL3GHfZ", "proof_sha256": "8346001fd035381202a6c099c3b31a0ae0bef38bbfc9d8901b6dc0ff921c542b" }, { "model": "google/gemini-4-argon", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950740-bKuPw464LYh8KMkRhUNI", "proof_sha256": "6ed7d6609c4c464346edf0aaf0943fbe24b91bcbab8a86086804b4725b0e8a59" }, { "model": "google/gemini-4-argon", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950753-1euVSjCWrXbpQtcVX5rI", "proof_sha256": "4fa754c50620308092ea94e8606dcf03b161ce1354917455c77bdf5720550e97" }, { "model": "google/gemini-4-argon", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google AI Studio", "run_id": "20261025T172733Z", "generation_id": "gen-1792950764-E5tthcTfVPCYGZyxpLEE", "proof_sha256": "f21ae80b8fd119f90e74dc4092eff75907dbc1f0320ecdba9c3a493f9534b1f4" }, { "model": "google/gemini-4-argon", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google AI Studio", "run_id": "20261025T172733Z", "generation_id": "gen-1792950784-Z2PZZCr27Gb4JtBQJeVs", "proof_sha256": "691bb682d211c565990f73edde1550bf1aca64d911d425ef049ab7bd2da15d5f" }, { "model": "google/gemini-4-argon", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950806-fLts5UFA0JRSyjkElQAD", "proof_sha256": "b5b01e3cb8e73b4aa83f7b1b3c182c83e2480b5912316e74e095c76570da7e6b" }, { "model": "google/gemini-4-argon", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950821-HAx4ooeufYfECFToK3t2", "proof_sha256": "c3ec3a2624e01069103ec9375dfbff77bf699e4259e4b28bd923dfa9fd926556" }, { "model": "google/gemini-4-argon", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950835-leJRuaMFVT6A1vPPM7LS", "proof_sha256": "ec630fc2b919660bdc246b053ad670bc277825c2b8abd33841c6eee64ad9a9a6" }, { "model": "google/gemini-4-argon", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950848-hAEVZLomFeIgp396ya2K", "proof_sha256": "4eae303eefc48d09413fca64c750bb5a55bce992d10d7629092bd991885a3434" }, { "model": "google/gemini-4-argon", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950860-ll41KzwYyM8HAPV8Rvl7", "proof_sha256": "093e90f211d59f94012ef6d488847dcb530d53d3ff69282630d0fe85fba97b6f" }, { "model": "google/gemini-4-argon", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950885-vVrecZyGOR6W6vMKDsXQ", "proof_sha256": "f3dbed0efb6d43affa4c0bba7d949db4f8257e5b5398dd03fb7c180eb06b5c29" }, { "model": "google/gemini-4-argon", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950897-IC9SwU4vH7W72M0AojYh", "proof_sha256": "f731f5384173b30865255bc13efb584b77a5eac975bb80355c5a735d5acf60c1" }, { "model": "google/gemini-4-argon", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792950909-SrQOMaeQDjvtxoFK23tj", "proof_sha256": "c10ea5f90eb1ffa7aed522f94a3b34547114d1fbcdd8537ff37199977b757d74" }, { "model": "google/gemini-4-argon", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792951019-vIVzH711Nc1wZ33jpdcS", "proof_sha256": "3adc94942002616dcfee2b202742f4fe6a239a12b75bf626e6e831376df1d51c" }, { "model": "google/gemini-4-argon", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792951039-rJnFdmHadDOImugLCPWP", "proof_sha256": "3f1a62030dba0c50eef0c018fb60a8566076482706c8caa19e91f2a770f5bd22" }, { "model": "google/gemini-4-argon", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792951056-HnQcAqtRveJ7pqHMrCFp", "proof_sha256": "eac3a76d5a0c5b6b869f2bd04866b2f6baeed2c0963f0b3bf132f79d850e8d79" }, { "model": "google/gemini-4-argon", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792951082-f5pZjZpO0mcCPX4E5j8r", "proof_sha256": "2312f0db79fb05686f5c1a62ae4cf729b6e2e27afc4d57c4ba38682aeab95fd8" }, { "model": "google/gemini-4-argon", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google AI Studio", "run_id": "20261025T172733Z", "generation_id": "gen-1792951097-CHX0FNzPEf7jPNKCwo1D", "proof_sha256": "5937631c2a9b6fa2f4f447c75767efaa9188f03e549dbfc010d265b93fb036c6" }, { "model": "google/gemini-4-argon", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792951128-UXOcJbCeCtntwkHzTr6s", "proof_sha256": "e70694db0bcfc6be729e8d8b6c4ffb1c5a38d8d71100276cbe77484da4bae5c5" }, { "model": "google/gemini-4-argon", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792951146-SVzfY3xFcM8nu44Kp6EB", "proof_sha256": "aa654e92aa8923d9b4545583560674e781a83b45998576d63c1b10d2c12c70c5" }, { "model": "google/gemini-4-argon", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792951160-K41cWUhpsaGpqMeodkXG", "proof_sha256": "d4f0d41193308205d25aa483bba1fbccaee2162f15e9a0d2d5724ba6f7010e2d" }, { "model": "google/gemini-4-argon", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Google", "run_id": "20261025T172733Z", "generation_id": "gen-1792951172-se7mj8ttCYUvxzDx6kOt", "proof_sha256": "f8ecf3e2c88d7b1f09f30204c24d7359a835fad72d16c623dd106dddea06a23f" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949253-WtJhxyFBt74H9iCOAHTP", "proof_sha256": "e349174dfce07529f48c878340896f577581d7f2309a2432bc7940e9718e6397" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949280-xZXK1OB6F0kikEZhHW5a", "proof_sha256": "b16c3500eedfcfcb4b79677c2005b436ae1225b207bf1e0c74d02d7989f85723" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949314-aG6857TDg9PGDaw6wJ7w", "proof_sha256": "f2a0a91a451f3f5dd77aef185d650afcee3dfe26c613438ff42bf7ab55a4fc13" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949335-q0xg6POmyBNhscl7cV5O", "proof_sha256": "cc78897e73b45764c4b6bb401c518b249235d9ab1a67f55abc980e5b0973fe61" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949375-x7ZHV0ldzcDRWBowqjCS", "proof_sha256": "ff684b4e451a8ae3ffea3885a1c9a7dea3719e6e1d89c5111d7c916a89a277ba" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949415-VzEWct3wAlM2b3G86ovt", "proof_sha256": "97b8205554b52933d2857c028afff231bf9a00476040cb78e6fd4a1f9df44585" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949441-MaCvUZLmVXVQOEcogQdb", "proof_sha256": "d7b430fb3f8e536db62761a01e5b2d8dcc01f947aca4fa8c959f1d71ad8f227f" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949471-psQDulGS7CRaqOK3xxFw", "proof_sha256": "f92ad17b1111da00ed696be33c174b363a96cd198e61c17efbfe61f784950caf" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949499-3DsmrFuE9vciAyLqidZV", "proof_sha256": "f94d47f873c17b84d287399ed13df17914c1870896441f0840a516cd6c7fe048" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "none of the above", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949524-nY5V77CEKLQYMnEbCNWE", "proof_sha256": "4e55d09b9382cd7d7fa3f7ca2d26710f11ef755992a132b36f8446c3a7b218d6" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949555-p86M9y4FLS1EuugvZAqS", "proof_sha256": "c5c6a9404c8e82bc83a65e43db6014a188031dd502552658e46e7a03dcf632cf" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949577-AKIH3dwqwKJSfPnObk4f", "proof_sha256": "16165d2b215fd3c508da80f1ed99c73c88cbc34f748e249d5a8fb2abf75c0f68" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949633-pny4gzsIrQXyDAAOIDKk", "proof_sha256": "0241a858ec0358d3edfa40e7a191df29d1b3955a6ef780811caf1dcac1816ad1" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949644-pQEfgi66GwgIsrVV8qPJ", "proof_sha256": "d8da5895631807bd21bb52435692d70f35daaddf6292f2fc547c951a45de184e" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949662-NCqHO4CXSxYPJWVuhuC2", "proof_sha256": "de5d4ade2619d53b1bd326f25381c248b044aff87431e842f800a46af60808e4" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949694-iIkmqzjORMWxhEHf2tP9", "proof_sha256": "837a7f0bf149bdd5d5c0a1f5b98412817cfc3bbdea90617d7400816cfc3324dc" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949718-dSrh9GR3BVUcT23E0vvB", "proof_sha256": "b4311a5e2b35e475daa0ea53044c283876c4a6aeecec6d8a8729778a6e8c361e" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949734-xq8s4M2bkwcEVewioUzs", "proof_sha256": "5205510e73942365b0a682740ee3436744e290152917c2eacd1e138ecb337146" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949762-RoHq7HSkSAHzaCiP92jw", "proof_sha256": "212c9909ea63cb50335f1f9210a7e59abda884cbd84d7eea98d93275ef3fa192" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949787-Giz4U18zra6sPdAi7g9a", "proof_sha256": "02a0caeb1038ad954d88b6bce5bc70aaf70d51385ceab9b42aa8c9f48dfdd8b1" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949798-cMTjfzj7hz9lq0Sj41mT", "proof_sha256": "f658e206d7a175603905acaf16ba1945bd315a1872a8d3afd082a3fe9a589e5a" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949809-Ru4bHNuaSZBTaF1PrdBu", "proof_sha256": "c4384a05900b7c8dbb2308a9592a08461cb31d1fcfcc0c3859d819be7ef9854f" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949836-2VgwY8YE6q9xbhKVdH7u", "proof_sha256": "4cbf40149be923089adc61fb176f0e823918df235384e15d7b490497e0cd4389" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949881-dImgy279wDrxyxAXtUz0", "proof_sha256": "fb9bd63170cd2dd9ba13600e9ea0d5d6607dfddbfa525efc9af621a2326e82cd" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949897-KmTQ3aZomVzyS4XOXXPt", "proof_sha256": "b0c2c0624f51982853e5249910cb698b750ce25536bbd81fe19e30879e341b49" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949980-Ru32rXOy36glss0v4P4X", "proof_sha256": "707bf41db12bcbb02410f044dc3d4ed177c36c794b119db203a97014da8c8880" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950020-UtiFw6ke8gjRjKIDr8ei", "proof_sha256": "7efa00acb40492709884b1d220007769dc6072c41355daa10fdef8d9b5f4d2e0" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950050-fSvqftmIQ6cmXv2uj0A4", "proof_sha256": "43dc8a19a105301201fb0366f7ca3fbd3a261ca8674bd870f42d3806d5c1bdff" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950076-P4PeW4ojCdHFuUqYBGQH", "proof_sha256": "54460cf117f1a028ebcac0c431b4d86812b863dc7799cdef307a42e028dcac84" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950121-cDACrqFBxW1NRDYFMO24", "proof_sha256": "f6956338bc2e197de3145b93623279f2ebd78b4ee5f8b2e93240a6a915b7b5eb" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950172-5MsE4pLtd52AecBTMYMh", "proof_sha256": "dcc023ca91d1cbcd9676289b547fc0c18cbbfe646a7564ce37efdd16e7ae0a74" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950238-V2bokgbCfieCJKHtIx4w", "proof_sha256": "b67bf8332f4f05ff8cbb73943ef1d2a4620dfa725ac62c1b21fc24ebc6c44742" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950268-AJucZCShnCgaE4GsnrNs", "proof_sha256": "f229502a10ddb80dea665c8cbc6142b1041647249bec5f8a41d224723dd315d3" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950319-OSBfXaYhZV7PVVtP6zbd", "proof_sha256": "4e929b6be82da0c220621e11b81a968c9fd9a13f8d5bf15a78aac94c9e9191d1" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950363-9pe5qju4VnBYN9nMvE1F", "proof_sha256": "ce588e8c0479e2b2dce63185bc46164313e6b47947d04a8ef5d1060dcd125f89" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950391-nJ5R1ymRWanMx88ippE5", "proof_sha256": "e7323f9acfa90e5b4150a21ab1dd500c3b64df2a2ba144052b76525500d893a1" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950413-0XTvlPYPjvoxG5B95G73", "proof_sha256": "eaa1078b37c0dd9f917cd4bb378fcc6bfb030499baa5e75a44ac66b8a83abaf3" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950426-80jCZfLNrqtU1cZyi3ZL", "proof_sha256": "c3c9495b251b4be4f5020a3d9218277473c24de7c17f31f3519f8b73c4ca3df5" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950438-GAX5NKHvcwwSgyPnnwSs", "proof_sha256": "58624506f79da5caa289741f4f349943705497e25b6b28b407a0815ccab9318f" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950456-VPjrpQyrakp1M9phYfEL", "proof_sha256": "5672df8ec595dd960f939f9f63abc6fd8f06d0bd749c706a10be39760974b6e8" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950477-EyRa9drUSCdXChvFGnYf", "proof_sha256": "6213efb1fcb23a78b9a53ae355af6d08e9034a1582c2a594089a672ea1600259" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950487-cCUvw3zO4s38QnBpwK9h", "proof_sha256": "fde5c0dbb802b0f1c752736db128fe3ed7e62603835c28196ebfa9351b32c32d" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950506-aHOYpRCQmEBqcMT0KbaT", "proof_sha256": "6fc4555b5fd7c52afbb6a7ed8d9ff9bf91d63f82530c19eb9b9c9a446d884989" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950517-m1tV0tHJ36tI5GG1cvBM", "proof_sha256": "3dad67b11ddf0ae3e2b31a93f2a6fba9af9297f67222ed20a56d812df10a84a2" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950548-phH8ZA6kwNOPZx5fiFh9", "proof_sha256": "8b56146475120b4f66760035320139b9676ec41ae278055da1a87a43633f8473" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950558-nxDNYjv8YiG1x7TGlLdO", "proof_sha256": "e32b589fa54c00aaa55c15c4c7510b912ce97c7fd28c81119af023db46544747" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950569-rahQ0twHja5CNpfW7IXy", "proof_sha256": "8b16d2789367ad55f5edaae10eb224c45e2e4e43cb339951a72c8c085d5ff9da" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950588-pAJamaNIJxVPL8GUQL8U", "proof_sha256": "f0270e162c0c1ee212f79ede07d5cf4c082188f0424d592a55d8f4e3e4617a54" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950626-lvixYe6Cr9osZw8pq9ly", "proof_sha256": "9f48624a7a54e143652df9bf240c3149234a5aebe16288d118559802d3b54e78" }, { "model": "openai/gpt-5.4", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950642-IjhPhHNqny85ZEQE1sBN", "proof_sha256": "b2b039bece1d21e8cafbb978337d885de7ba2507627d0353f8ab6b69b57c4c9d" }, { "model": "openai/gpt-5.4", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950653-8zQw8snmVR9Oodj3YCPK", "proof_sha256": "e79170868e7f8276ad1d19243e00df5b5f47ab34e4573c9e53910c75b0bc2681" }, { "model": "openai/gpt-5.4", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950669-X1PloBOXigsORuFYSGJ8", "proof_sha256": "8bfdb4f4d348af3ed04ca5819762d3f908e8213fbdfaf1d1436559f3dd8de6af" }, { "model": "openai/gpt-5.4", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950682-Vn1LxfS9XvBso13gDMY5", "proof_sha256": "39e964b50e78ec0ed99e9ec2c29e1f2bfc6335af2d7ba825b28ab497513f7488" }, { "model": "openai/gpt-5.4", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950696-F50wt2iJiwCCxjXXYOWB", "proof_sha256": "04e8e28b37d51d30014cbdd838bb327f88f4e73cd376b16260aebea3edd040a1" }, { "model": "openai/gpt-5.4", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950732-qB49jxFXMIhmVh959YTp", "proof_sha256": "5c4669ba51725cbb8376acb14915f9c0b8cf65fd3151bae245fea5985cd902af" }, { "model": "openai/gpt-5.4", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950745-YCeiCPnREuYJNeKdt02v", "proof_sha256": "1b79340ad1e519fe35edc30214097e3024f51c5271b75b91172244cf88c54774" }, { "model": "openai/gpt-5.4", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950773-p1CTYILnUO5XjXWg3r1G", "proof_sha256": "9b01d84d33d33026c0712dde9877639f5f83c9a32af4dc7e27f567c550da1780" }, { "model": "openai/gpt-5.4", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950783-0CeruSUViC7vg0d90BLu", "proof_sha256": "6bbee9bbb4ba20471a544ffae6a5308eda9a1fcb0eb3516c3de6791e6a919883" }, { "model": "openai/gpt-5.4", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950802-gAfUwqBU4E8DsRweXEgN", "proof_sha256": "cf9ea85944bee065f12c529f7a0c7e7d42f5ccbda87d653136a22a5e373adc18" }, { "model": "openai/gpt-5.4", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950822-RnnGT26ZAvWxpbBZGhbV", "proof_sha256": "06de0a3e50e72fe4536da9670416ab3b1a7a32641eb84b91e60b422a6b18213a" }, { "model": "openai/gpt-5.4", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950832-pEIRuIzN6ElogRThJqZA", "proof_sha256": "c878ca286b0018b13eaf70dad3f8e8cef5544015d97705c79262261ba0864a7f" }, { "model": "openai/gpt-5.4", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950841-9SL2UbldkNBe5dIMacqa", "proof_sha256": "c3335b8f5acfa29ec5aa573b1021508fb65c4cc282a64b9d03c22af50e8c83d6" }, { "model": "openai/gpt-5.4", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950850-TCAyxGPxLo4XMFdWc6Mw", "proof_sha256": "bdb0691c73f7177fb7324b5fffddd9c15620b3a1e2dee78678a7023f6063c832" }, { "model": "openai/gpt-5.4", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950859-vnnNfD2bZJfOb7LOFEKB", "proof_sha256": "5ec8fa036ac53bacd446a989539da8fb4e81198e4a0c1bb5b030ae39c2c903be" }, { "model": "openai/gpt-5.4", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950868-IyTpcU3sSmeZpciWLGS8", "proof_sha256": "efedee80b025cf07481a4c72985bbfb92d7413455456db828562e7f1f04e47ec" }, { "model": "openai/gpt-5.4", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950888-3ORZxU83rcAfYjBAwcaS", "proof_sha256": "d7dbb1fc38023b804376e3f1e2a0ecfc8abaeee1d187dc22fabd0e231937afef" }, { "model": "openai/gpt-5.4", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950897-F1umafywuFhKD0pVlQxb", "proof_sha256": "7f4a91ac4a4bec063f2c692b06445871d1eb64b2d091a8f7877fca8511b3bac0" }, { "model": "openai/gpt-5.4", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950907-Hr65baaVBBX9UwymXSvn", "proof_sha256": "eb5ed5f0e561c012b3cd73d0605f78f42e84753f55904bf6085dbae743b97814" }, { "model": "openai/gpt-5.4", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950920-aX7G8MlliX2TrAOVPqxi", "proof_sha256": "492f6b8e1137a2678534927d2fada5cbee5293d880a15beca04937eac4b08032" }, { "model": "openai/gpt-5.4", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950930-et2eIoD3g7krTo43rIgf", "proof_sha256": "d6c00e03bc22f575278c96c33624f96443a7f092bfaeb702d57324ae83ca88ab" }, { "model": "openai/gpt-5.4", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951013-27CjMIzEqKGCZifp8Yyv", "proof_sha256": "1cf2bae0f5c52987940c4033f43b790d9e6661823fe8e5c9cd5885afb8a1794a" }, { "model": "openai/gpt-5.4", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951032-nshjUjSGwaYvFy1EkV91", "proof_sha256": "e6cae19f0488b0f35e179a65341e258aa1a97b993bc379d07b2af82271f2151f" }, { "model": "openai/gpt-5.4", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951063-LfCbB4fEfbaigeKX8PTy", "proof_sha256": "1020982664fc467408844f9b86193292e74d990d2718585a9826ab8fd3756b70" }, { "model": "openai/gpt-5.4", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951082-PyY2Inj3Uku74ySoIZR3", "proof_sha256": "0f32188da8ed7662428c89fc33a079b67e9f5e9e18c143e56daa11c2066e8453" }, { "model": "openai/gpt-5.4", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951096-7txOLaKlxGpqxxLg8oM4", "proof_sha256": "3b5af728b05a5d01aac8271d781821ad65b3edbe50a0dc35a0bc3740fa231f22" }, { "model": "openai/gpt-5.4", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951107-OCWjT1KrD001mEB8iHmd", "proof_sha256": "44142fc0a375393f899f9be448211b2577264a650d4743c4b6961639d9a7f18a" }, { "model": "openai/gpt-5.4", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951118-Fz2iJLukIZFXRnNvYhnA", "proof_sha256": "9cd6e4e328dbf77b6788fe2f5873e01aad1c818436735a2e5333fb53560c433e" }, { "model": "openai/gpt-5.4", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951131-zseC1TB2qRnsFSHvWEu0", "proof_sha256": "9b0e0a43b05ba3202e35e37162ab31cca0e34733c62b5b345f558e307bd41081" }, { "model": "openai/gpt-5.4", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951140-LbOfDDX1eIrDYfOh8kxH", "proof_sha256": "b168998b00023afbc5096eaccbab2c4a675c3ab206a8d05df0b8442bd5c8a51a" }, { "model": "openai/gpt-5.4", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951149-wa3zGAsyV6XXmO7Pnesw", "proof_sha256": "6afed6039bacdd616f235701a3f260870550138acd2fd13cdc2211d2a5ba4bd9" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950611-lUe8cxulQvhRpnpRhamM", "proof_sha256": "9ce5a131dcc3b527ea3be4523f387a8316bfc7383cdb36d2ccb9d3115476eae8" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950624-Me7ZRnwqVB6dP7RYZ1sh", "proof_sha256": "f7d0017fdad2b170edc34822a5194dfc78a1c90c5835a5ffc628facd514c86c0" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950636-tt7p4x4gFQ4ELDJGzxOA", "proof_sha256": "c4bbadd683d1aea96a9f5c7f49b2eaf5a1a0bd5df6a818893e81642f11f8021d" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950646-dToH5hInvhwvid0dWSYd", "proof_sha256": "43e82611a6f7a897837fc3bd6224d3c47cec49760d1c513ddc8fa0b413f1dd73" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950673-ASgfywHrI8px348d2NiD", "proof_sha256": "849fd966bc36c3689a2e746734274d203f656b507555d0d0ae1ce5b26785b7de" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950691-Uptj5zFTRFWUYkrcgOkt", "proof_sha256": "c4c61587b4f142c67de3376c8955001f82d997dfe892f46f74333319f987803f" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950702-TJUQXvcvllAhLu1KoTau", "proof_sha256": "56ce6c6fabfd27ba1d02486fc8ce41988611747266e30c0c060261d06f692bc3" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950711-PRhpsgdxweVqwGwqZmN3", "proof_sha256": "4ad997070030b259828246b6ae7eb83b264a2ee83f4cf2ae53c3ffd3fb7e9b9f" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950724-FM2JcZF5ere1mKK5qrMd", "proof_sha256": "35bda956d8c456936d9c0278c24ace5c3af9c78fd6848a0f9c016ccc3d845aea" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950735-tHMYCMU2yoDyPlOtycPA", "proof_sha256": "67738185b1857a7a1c1e5972b3bb1466f4f1c83816da3bdfb8e419d2dd78c00e" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950745-J9l0FCjBFdmjDrjsPiKj", "proof_sha256": "108944fa26bf267a498b0efb4a1c79ac246c8be85c32d1597a425efd336c5640" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "residence time. the residence time scales as the deep-water inventory divided by the river input, which is dominated by vd cd. taking vd ≈ τm", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950755-QnrmagST0dJ0LWpfyaED", "proof_sha256": "6f0a91bdf0ac44d69e68c32509c4f545fe495aa72022c8b43bb6210b9b79a033" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950791-mxhQcVoaDXrUZ0NnGyk1", "proof_sha256": "85f48276930ce7f57478c38abf89427fd5f5566bcdd551cfe068292584e17f8e" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950798-eggRWtQSnmS32LZbvwOk", "proof_sha256": "177751b394ea6dac490c2fee819e39be4cb42d0ddd235481032415b4327e5349" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950820-F0pM1JEXSk6sH4aqBMh0", "proof_sha256": "275d3ff7091a5d130654caa2697d8de55310a420a58c806faf45e67c653e2057" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950833-SIPzUgaAYmVIwfjlRE3l", "proof_sha256": "382a8c7623dda8f46e162d14fbe2ff1d27b097c3057e9a748cf692aa6bfe8172" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950845-38FtrBEJ3OLjmaN8gdL9", "proof_sha256": "75cf4bc092e6c9e583cf3829295e6819da50a629857a7559b6ecbf2db343ed9f" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950858-N3TCfpmBA0UZUg95tlEI", "proof_sha256": "b25f41189edcf0bae462df2192f8e55e67d7a6bb425cacb0fa9dfad0f24e6645" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950868-GpN1tDbfOol2HfLPhIcc", "proof_sha256": "f2a005a33b4f0a8d99027fb01b6c1236ea4de7a34e2442bea80a776c2c5d6c4a" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950877-ytfd2nSjDM9JQHG2wnUG", "proof_sha256": "ab81f6922579b9024208f95f38fb40fa075dbcb27ca34ca6b4a7ac7d0d162af1" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950885-M8TxXVA5v1YZzLGL1q6f", "proof_sha256": "5bccc230f73708d0a778904470f6dadf647fff999dc2bc4707df4d1441a164a7" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792950895-eV2Jp0cI4Za1R8em1YPD", "proof_sha256": "c15610a558b7d48e7e942a299f4e52d009e1ef88ddfd1300da251f575eff9454" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "the problem most likely intends the al", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951013-h2LhTiVxwCWhyCFUZnmE", "proof_sha256": "33caad0914c3a4888bec2114310b919c77f6e61a0fce0cbc25fc0abf0487faab" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951046-QBGzzbMYEZ40RwAOLXh4", "proof_sha256": "3c8c332306341bf81fedb94423324b3c68cb80d58e03a985dcc3874a3acdd7cd" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951055-BjF0KCtZqXhA6M9uIIC8", "proof_sha256": "5b49394a0d2ce356cfc62973d163069152b70c9ba5e6251c061a8696a05180f7" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951068-yoXjSSdW01izOopLcFMB", "proof_sha256": "4dfdede45132bfe743955d3dd0c054a36bd04cbf329719c65b6be74477a285e1" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951091-novD7ykdaEbkww7u3vGG", "proof_sha256": "e287a37c53277cd466f8a427f574b93b1a12f1d8d6e4ad6e15c292d08dce81e7" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951102-Vp2WCHmULVjdJYcn2WBi", "proof_sha256": "a10ec9463da0ed93a152d1f895c1790206469e2a6f60ba89f8a230fc45d3db05" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951125-4Cz1Y5OvEglzk4RtT092", "proof_sha256": "c77af2e11048bfd7a866a98a42ab5bbf2615be886637e05f957e629dbae48a08" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951138-fcET0n3Ob3LBWeVjGmgW", "proof_sha256": "17319891bdec824ae5437d09fc3223fdaa8d38869f61b4da688274c0c640af3e" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951157-zOIApqJ3QtRzaEhXd7S3", "proof_sha256": "5a939799c82c7f5d4e82cca9e8acf4a5b2032c122311a0ef85e4b47c5182701f" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951168-1f6RmiZ0gEN5jkxlf7q6", "proof_sha256": "873aa52ea9f27477420d33f346f2e7eb50bece5ae75ac742275ed9d641f27c43" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951195-6UBC02TumxY5vuuKjNJt", "proof_sha256": "19f5fe2093b8580fac423a5f33c4d089ef883911a7388435bb420fbf0a6c0eb0" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951209-059JRaPmufb0skVo59es", "proof_sha256": "fa236843bf5874cdffee34cbbf169831b0d40932fea28e3356bbd873ecd3ebe7" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951228-MeWOCJ16FWc26mQ6s164", "proof_sha256": "c49a33ef549f9376320e9c06d710b534dc2ed9955274b44c9d6c64b177fa60d3" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951249-OFzFZeMRIFmMtTRiAy8D", "proof_sha256": "b73d7e63b6f4ec69b15d544c517b1cbbddb390be91c22bdae780e62283cc7fa4" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951258-D3UI3XDTR46eYLZuqPrN", "proof_sha256": "e510694ae6cf7090200bfb6f6e4f5e90e91fcc90f8846329a3ab7285b891ca7f" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951268-WTQ71gFS07ws1SfT5fBd", "proof_sha256": "2bf806874389b98626b8af7e976814443e14ead930e10b6b3cc79ed5d667a408" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951286-Pi4TMhAvmawxCm0ZxumO", "proof_sha256": "2df9ee4174d0810e267b432cc6cb931f1a82a0bf7fafb45ca6c1de3cbc345ec7" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951299-dw3wXBsGoPrdWtcP4uym", "proof_sha256": "d4ec9c9ae78781f094754075929cc57285027233107c9b817b36122fc70ee1c8" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951308-xnlkQ0oHEZcsMpuyMeGB", "proof_sha256": "7fb8152f1107890946a96d9b82e5f199d46963261cdcc111ba54ab10e72c18cd" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951326-Jnvke7unqGmGVU4sRu2y", "proof_sha256": "6e955ad38dfed00d4fcce3d12d40bfc24a13b2f6a593ff2b9d67cf5adceb1e3f" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951335-mF6GVyLEOq5tPSFVVTS7", "proof_sha256": "ff652aff0f7be58a7826c4006c8bf1120a2018c9782b9505b4ee2abdff2aa7c4" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951348-zjihGvjO8KDetPUm6jh4", "proof_sha256": "e7be4a30ee3d3a035e4b0a892597bb600c3bdc34b0dfc7e8a472d53c280b2ecb" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951362-xExorGSAGfrRNEwGnrny", "proof_sha256": "b51fee1c5bfcae28da80125fa5e192b1a1f075d7afb5133f90df4e858586b3c4" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951371-DYjebMJWT1Y2fxzfp3sC", "proof_sha256": "c585ba87f62875b138ee223f43af8f1c2372f34f3ff86ed1b453e5ea381401a7" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951382-f0qazG6JnPG7YvD1CrOs", "proof_sha256": "8a993585b8e8ef6eba1125f5229181306a9479bf0079cc838bfdc8fb42592772" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951392-l0xgeq3IPmkUepEU4Um7", "proof_sha256": "eb2799d04d153f1a886a814f670fbbb40391eefbd3a6cfc810a4fe2f361b419d" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951417-6ojs6OZhjtqXAcfs6xCu", "proof_sha256": "421c4e36da3e31f0fb410f37d8327e4a6f572cee2b0017087090dbfc3426e35f" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951441-K5hHNPO9QBJGx8o1It7q", "proof_sha256": "745189ad8409898e949fff3027bd971e5656e51528e0dba29b67ed77396304f8" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951462-xLfKzXSv7VrZJ9dkEGeX", "proof_sha256": "06941161ec37640a95d65158a92fac38f3d5386d824767433381789a4b13b8b7" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951475-tBB96NntZw3BPg2nAtqg", "proof_sha256": "094f1d5375db5b69c9229bae0f5839c8fa24700ae5bffdea2595ed423c5136ac" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951497-ZJdkxG8KpNbwPejO5SL0", "proof_sha256": "0ec3c17c7ebb955cec3c512472ff83f35f4f6986eb31528b8eed5b49d0e25455" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951518-1l5YzLOtNRtKozC3LNrs", "proof_sha256": "3a77c0c7ee36a4f4cf1cf2db26c670979ce3c158003ed6016034aa383a0151a9" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951531-5bUR6Hq9G6MefdcLRYFs", "proof_sha256": "f8b183e1f735929f615de8627d128c473ec7439f967b5f1656f4bc51667ac2c6" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951547-oO4scxk6H408ZckigjG0", "proof_sha256": "10be61fd6504bc50a72bde2b0039c4600a83a2d910f9b637426823498bb4f5a4" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951562-vrIE95nTpR9jpMgtvpUI", "proof_sha256": "0ed35d737c3f7f0e56ed24eb529edaf615e2ac68030fdfca2aad9f4008ce00d9" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951573-pLMlzdiw3uicEdRfxftD", "proof_sha256": "3b309338efe75edcf20646c58c957cee5eba4b491fe864a8700e3b0d5965013c" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951582-Znrx8miBhp5px9NIeGnG", "proof_sha256": "e48d5f160489ff8ea0c36faa3802705890c2548cd145312f0d8221719290bd4a" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951591-gChPI9EXACWB7qNCMWHR", "proof_sha256": "4439e174b5248afbf0ac82968c67a7f5c753434e7a0037cca985999b52344c48" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951610-SopOeo48bz7euNDXXWqN", "proof_sha256": "4db4e01b404133ec0ec36b8144271a69751a4f2dad4d73512596cdc78d64bdf1" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951619-mDTQ3WcAILGSLXlW31OK", "proof_sha256": "3cb9deebce081815ad4420e975bf475ae424e8b1edda02af63fd539f39599509" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951626-h58kieogopoAnhOkYeGM", "proof_sha256": "003834689dc9b35bc811a9f8349e4f827d7fc2a75d1a5baaeb6b78727cef62e3" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951636-I5SSlgeSau6xKVQzvxb4", "proof_sha256": "db64a3f466a92168de3fa681e51583b4718987e0e6869eb5b7dec5524d98e6ec" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951645-YrNv3nGWfenogapVD6iy", "proof_sha256": "b597390bcba9705fb1ea3e80b8727ac46807aac21c1ba3efda8b64247a87ba5d" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951664-3VxVQkKK3MMw5qV1eIun", "proof_sha256": "cb0f4a0cda6f29e6726b1fefe401117c8e4b2e4211ad9c30d6ad8ade4edd9b1e" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951681-X4npfeguC6abcCwrkh4X", "proof_sha256": "b71e749f39237f8d84b9d509f3490446d0d5bd25f5651612566bfcc424463218" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951690-aHkbrPBtLYdvnr8C0os6", "proof_sha256": "8a0bf9479e7494a3779bcdfebd3aeb825ecd5e0703e2b9d75e1015c6fd5b6a02" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951703-LrqVBu12U0Cynr4ZyB8m", "proof_sha256": "f262c08b6ba48082eed5fef96655b24438887abee5f2ada7f7832fb4a3306e39" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951721-fzfXVgChX0K2EZ8QMtuG", "proof_sha256": "60c92686f22dcaa47547244897c7af6fc5ca12922aadfd9a29c82c8757d9dc96" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951730-IJmUb9tpnao70DMT9jLi", "proof_sha256": "ec5c198482006393ab383dc91bc06f6f2f70bf76fe8d9a5abae2038326926875" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951748-VKW2QUbnIXlZCJMiQzub", "proof_sha256": "6f1994c728aaa5e45b51db9d5daaeb5bf809e8199a9a961e9a49053842b2bb55" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951771-wtBMRaCY5le8MKYPWTjE", "proof_sha256": "7c4bab81daa4fdf105e28d2a08f0b181dd83f681532f377abaedfa26b9e33e3d" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951779-D3bQ8Aiops2W8fDW5oT6", "proof_sha256": "cfa4f283b51371bdf6c220460eb2d930985a7a83f023b5b491f6a4fb5d179562" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951787-C0La2Zl8VoeM3zvjkqz5", "proof_sha256": "68e6df45edf227c14a15d7e32997a75fb6e6c4fa16011ce20f0ec739105cc09c" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951796-kXLKDsRClWGqmR3B04zz", "proof_sha256": "743ca950be4d2b9efb4a691617efcab00f55a3763536eba238ffc55da4ad71c1" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951807-RDZc48KhoPGlu9AQJr1J", "proof_sha256": "1a7cc6285c60857c7e9ac0e5c67eefbb16552f3cc50f0d9a2b2674d0d4d4af5b" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951825-n38RT4uQUYIum1bE1cPf", "proof_sha256": "7c3b04bbe1ac2de0d3d76fcf2737276b00cddc48db036bcb06eb649b76276fb3" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951833-V3x5xN8Z1eQrgzG4JJ7p", "proof_sha256": "b5597d2d029333936dd1ebce423f2aca208639373dc4e775b9210cce5d217d73" }, { "model": "anthropic/claude-haiku-5.5", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T172733Z", "generation_id": "gen-1792951851-KG5FmnixXTHOhhHyRTD0", "proof_sha256": "8523b7e073211c077deeaa4c41b2f3dd43e0092f9e0091db4e97872ded00cc1b" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950731-LhTFpDPCtchsR5HPk194", "proof_sha256": "4166253ee93c01127119001ad59b510c3c0f713447ccd0886f3bae6366b4975b" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "c and e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950741-iSWuT83xODfgyHuittMe", "proof_sha256": "bd081f86e60a5ce02f810ac58f013961f9e1a3271dcae34b7385360fefe766ee" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950752-UzMlDgBvwvEAmCoKww2a", "proof_sha256": "4ee358713d233d411cf7e6a8ca76f914349a5049398863f1a191ffcf3a48094e" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950763-uqERMJgjpGmOxY60t40R", "proof_sha256": "b10b2a19f31efbacfc0228fef163717a7ccffe67665b77525f90ca8db5abff23" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950775-Tkh2FwtLBaWMhWxrfOtg", "proof_sha256": "3a5902dffdb259f5a66ddf5142f62bcb7c5758144f71bf6cb89ef3672e370846" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950789-OSvvV1TiwyJr9q2oy9kC", "proof_sha256": "fe1bb4609ef09ccdc40e37b7a6dbba4244ed15942d9f785cfedc043a1d6d975a" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950810-WcK3P2TwbKPhC0ds98aV", "proof_sha256": "a701499fc27f2449bfc2b91ea3a4ddcf6b0c75dc517e4d00cf1ff8dfbf774b84" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950829-RnlhborGfIwuWle60lUC", "proof_sha256": "e11f603803d70a9a5e61b93ba5fa37a6cd2a11cb27bfc2d3de8d6ead010676bb" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950839-XnVQRZs7QYZV30AgLQBE", "proof_sha256": "5626568817bda293e3add14266835310dc9759eea286579946425b2be4ef9ae0" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "the expression equals \\(\\sqrt{3}\\), which is not among the listed choices. there is no correct letter to select", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950848-GUuJF1ey4X9sQlcnztsQ", "proof_sha256": "b15cb565d8d720a1f655dd0517a5c9776ec048d3f2d96adfe7b3c9db743fc847" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950861-6QuUifhdrd9sOV0ZNsFv", "proof_sha256": "b791068af0fa6392fe23520720e0cccfbc30d8363ab0026f7d4ea30c4f266687" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950870-d4AZbzu6kZcF6qN56Eub", "proof_sha256": "38337fc54f731dea5dcd269f8b285511262baa6e3a8b2bcecd031062767240f2" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950881-pfnZWEmh3zT2YCEhVWZ0", "proof_sha256": "f79f35dddd7962e96d90ee75dc86aa3212a3af334486d31489822a4418c3fcc5" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950891-XPu2d7S33K0gmWAbXIaP", "proof_sha256": "6999a3ebecb0f6c13c296848247b6c90fb61452f1fc149fddf4e63512cf16cd3" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950901-TuNk5zok5TLQxVeDZzyk", "proof_sha256": "bfba8e9d6c6d954620adac6291640b28f2d53dbee32f9a9d58be1bfe8bea416e" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950911-2F2ioDiMWN2NLn3AwsD9", "proof_sha256": "c14886416a34932f7f5b6291780def54be03e41bb5a22ff57eb18330bef60b8c" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951013-PN6QRpco9urZjxyNww90", "proof_sha256": "fc948b9ef4ceabb443c1258e54fe99a093e5fda1b0d68d5074f39054e3f8444c" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951031-09R7PXUEv3IHf7pPgTgp", "proof_sha256": "0a09ed28942b5e0aab0255788a608f058b66e9811caeabdb71d990b3a0be23ba" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951042-0zgdPmmE8C6Tepg6wdUh", "proof_sha256": "4c1dcc6eb39741cdefa94e2733c5c7b709fdf4900de791a0319d2269196b6338" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951052-jIKqmZKm5AqhO3aSxnmJ", "proof_sha256": "3e0d5a2256dd309b554159410a4ae9f8d8c319beab81d1569e57cbb44467b1cd" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951070-XaVmzeqHi6g4D6IQUEcU", "proof_sha256": "0bd61a6b2b7e11892ada7c2ad7c445eb5fad66b4e773f93c7871558e4b9dcd4e" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951080-SUzLjRLJI8omZQjlQ6dG", "proof_sha256": "543368c7580b33e28a5e9dfbde521903042a014526487ca6359b202f1d36523f" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951101-gh6sWbq7OpyzrM9PW8i3", "proof_sha256": "70cc164da467b480d8f302fc06d753ae39ba012c396ca8e4db0f3846c1b8c82b" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951117-r3jMW2uJ84vCgULBfspZ", "proof_sha256": "c0131b444e57d48e8eb535c9fc006c59f56cb3e5dca70620cc9f6b59cb20e036" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "the question asks for four thicknesses—velocity and thermal boundary layers for both water and air—but requires a single choice. no one option can represent all four results, so a single best answer cannot be determined as written", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951127-67dz1squjzxHRJi23VrV", "proof_sha256": "7a5a290d416789c13b34b5d70cbe8660e87ab61262e536f0e5cbf34ff180a119" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951139-5v7PGVRsiTGys9UKsz8C", "proof_sha256": "e46efee49cbb535c23ee9a6640ded0160133d9822467b4ec4d9c4cc256eb25ef" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951160-ZZQzwSCAO7cd4Co3YuIQ", "proof_sha256": "e508f17c8a005cdf0c70aea3f0b1a06ffa68a320f2e25be2d57bfa95ad6d719e" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "none of the listed options", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951171-hMnegNyfn0ah5QyUdJkG", "proof_sha256": "4c06cdc55bf45bf48d63e8cef05c489b2d5ee76e457447471cc533a1b824daef" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951189-uoCIWnv9AUbLPylJdCt1", "proof_sha256": "c34b794d427a5ec03c7cb7dc7b9cd76d7c21cbd811a7355a75abb7068c1402f9" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951200-m5dQqPjs1OF1n1LW45Tr", "proof_sha256": "4766a0b847f879690cacbf19d6088a83ee4a3ecd54ff5c13f2e213ef50c864fe" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951213-3mAHyJRKtzQwmXNeN04Z", "proof_sha256": "3f4ab6e29772ccad3114ff5656dcb84ca11b5b708c6983b2b72e64548e6c8f41" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951224-Za37QzEmtvss0TA1e6Op", "proof_sha256": "f05fd1ff0d2c851e0334e8154eda27acfe767547cd92b780fe1a7e9d350b3a9c" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951233-5dEqhiZ9CRxbULP9oaCC", "proof_sha256": "556707ffce57f123d2a2a62f7eee215f5b6d524aa65b9230c133b4d76c1dfb33" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951244-FmV1cdBD7553l0aBqLyG", "proof_sha256": "98cc0af670dc87dda5d378b3c1f513a456eed5743774df8a1c77a7340e971e0f" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951255-K5Nj8pX86aHMn6uflqe3", "proof_sha256": "1a4614750d8e4ff08cdc91a09d4a3cdfed3ff6d191dbb5601976b26c4f9715b4" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951286-SqP3TXLlufvVl9vOs8am", "proof_sha256": "033056be42ee895caa3937397a5694b75fe8e62789a0800c98e0d984294d5fee" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951296-rNqogg5rDknkoOdPaz5h", "proof_sha256": "483ee73b787fe52ed5b5cd653d3798463703725d272e1c20c6d89ef14264ab93" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951307-azyJW7F9MW3MRu0Kbgv2", "proof_sha256": "37227d42743eee86879092b2603276db599d94b82eca2f4444d6dc201d45490c" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951317-NvOl60g07tnemwJJte09", "proof_sha256": "0d37278847b3345905b8e8f7d83a4c82385cd46c04d476ff29c306343cbc2c64" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951335-RepRARTBZoMu6Q5kaSpb", "proof_sha256": "decd16e79810b9e147c09d26a32204d50bbdd3d99a1468ace292c2e855251ba7" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951345-h0VUQvPcuriDsR5irssY", "proof_sha256": "8006bb5b48b2b2f70b494450f66dfef045b5dbdef878fccfc8eaae114a026005" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951363-HrvwRxFyTvDgPoZfwGW9", "proof_sha256": "e4a22a2b49430786ff8781e78984a0146bd0c58b328d8067db93ff7af98f03b4" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951374-DatqnOZamHYJC5mALtbQ", "proof_sha256": "a51cda8ba3e282270ca8a9e1952a78dfa0a29a56c6c13fe62e2029e4c6635957" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "option a would be closest if the stated specific activity were \\(1\\,\\mu\\text{ci}/\\text{mmol}\\) instead. as written, the question has no correct option", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951385-boyf777CJTzKp2TfFKGp", "proof_sha256": "a904bb1770755d61112f9e461304647e5f090f2b91ca1d55c6a108b55e611f44" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951399-L2emJvb1K0PGFFubvrZn", "proof_sha256": "9ec964510d79dd2e592f681a6e2d44646b438966fab08f544faf36e45291d589" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951417-EJkbBA6TLppwF9vZJS5c", "proof_sha256": "8954a146748acd76bedeaec0219c225a6928919f146735334002df3555a18679" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951436-HdT2PbPa3TXWDuqAY8mk", "proof_sha256": "1683877c76eccbc7d64197e78987ed5b44cd2a1e1aac04a6baca28e6adac3274" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951446-rriEyiIOfGnpJ68NCbhM", "proof_sha256": "2ba717812da16d59d120ddd13fba2f4fe7a4455a7ab6dfc10a67f42f1a6b3187" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951455-WyQlad0ifgDSf1ezxo9X", "proof_sha256": "3ff20a639085f5cce4f3c1d5539efd423fe91054cdc5ceb1efc23b873cd8dc8d" }, { "model": "openai/gpt-6-sol", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951466-zkS3DcwxUH80o72NL71e", "proof_sha256": "39357ad1cdd4aaef335e4b5ff9b1d040af5bd172c38a251f9e0a03542b0a68a2" }, { "model": "openai/gpt-6-sol", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951484-EMYvnofeIN89TsCaEHGD", "proof_sha256": "1a9200526ac8c796cf0487b54d2e515efeaf902a18f5f146fd75408c17f11839" }, { "model": "openai/gpt-6-sol", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951493-PulkbKZL71zJ6zr82JGT", "proof_sha256": "a802daf5eb13ff427411b67787cd773c016f0013d53f0860387c035553849d65" }, { "model": "openai/gpt-6-sol", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951512-3iiFfDOI5iPdDCBUtLFa", "proof_sha256": "e1eb2e480797db2386c61539c4c9b4b5090f7ea4adaa51903de610a279deb95c" }, { "model": "openai/gpt-6-sol", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951522-jBBo7eIYmiZnr3gsbzhp", "proof_sha256": "c071c2f8d67dfa2cd627ce6118221c1a864ea568d6c0a48932c5c9fedd959e8a" }, { "model": "openai/gpt-6-sol", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951531-VOm5bV02lNgSFoZOxzA1", "proof_sha256": "17412f3f68afa03e39088dfbbfb89dc26e0cdaffb6810ea5c53b2ed95911449b" }, { "model": "openai/gpt-6-sol", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951544-U0BVhN10RBqeISqqTOR7", "proof_sha256": "9992f0768a041f26d10e5969c9b40d48d76a35dac2a9865f5736ef166cdc0dbf" }, { "model": "openai/gpt-6-sol", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951562-K4qy0waoPfQHrWeFI7ok", "proof_sha256": "84006bf49d158b510000be03738f0c2436e2e6e1e1b1aa2bf8fcd0bebad0f5e4" }, { "model": "openai/gpt-6-sol", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951580-o9fVj4NZjXhebKE4M1Z7", "proof_sha256": "ed9940a374931124afa9e6c7cf332f177e61b71cbcae3bf1ebb8a3a759231580" }, { "model": "openai/gpt-6-sol", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951588-cAcSg1MSCrVJy9GcDAta", "proof_sha256": "52e169c916407ed3d26dcd495f4af63bbb6962d8ba842c152cade31f66b7dadc" }, { "model": "openai/gpt-6-sol", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951597-t7uwFHuR6dfKuufYuHHD", "proof_sha256": "ed409354da8ea7f8a0aff2cac0d6c3cb0de2b750d7e2897d61aa4ae5178bf280" }, { "model": "openai/gpt-6-sol", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951605-1qntFUHhFXSjaKmJnugk", "proof_sha256": "2898a334cf3f875216cc70b70b3143f172e0d736919b55ec2aef108097751daf" }, { "model": "openai/gpt-6-sol", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951613-Tlu5B46F6dat5YAoPpLG", "proof_sha256": "fad67ea02be879173e46eb954dc1b645c8e75a24117a52721f0c62193a31ea71" }, { "model": "openai/gpt-6-sol", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951621-yhitkLCVvlxiVTk0uZm7", "proof_sha256": "3036fe6d24995f18e5e26346e826a59aaa62d39b6df7f5b8c873547202bb9aed" }, { "model": "openai/gpt-6-sol", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951628-w9NtGok0EmISy6YRvbBq", "proof_sha256": "6fc05961450ecc02b62d4d30f6a49674227fb199be503ae4e4afe5b9ef3f7787" }, { "model": "openai/gpt-6-sol", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951636-2vvKhdVnbh25izY5dYQS", "proof_sha256": "96daa6caa22794e9342c863a4e8bcfacd1c3b86e961d9f0cdf178e6afb34d872" }, { "model": "openai/gpt-6-sol", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951646-14veBsppR1JBgtwGfD3F", "proof_sha256": "75e6402c0e5afd842eeda8367ba9953555cd98311e73267f1d084fb84a4db549" }, { "model": "openai/gpt-6-sol", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951665-g0ZEVnBzyW313LyQi1FS", "proof_sha256": "095cbf6abac4bf944a261a08e9c1442477df94e9749f4cace2fdd9172087a8f1" }, { "model": "openai/gpt-6-sol", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951672-pjZTB7EOSzrApl2Dj1Wl", "proof_sha256": "6dbe2d713763e88ad5dfdc436d3907be2a3b58e3684bdaaeec6fc6696ede10f9" }, { "model": "openai/gpt-6-sol", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951684-byfwKg4vrU1zIXs8xwis", "proof_sha256": "afafb0fa431abe8c0678975dd003be91bd28fdbb843bb7c5e3fd66ffd1733695" }, { "model": "openai/gpt-6-sol", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951692-9GbmVpWs8KVExWjsOSJx", "proof_sha256": "91ef41b04988f7edd88399daca42225e1b75c6ab080fad666b7dba06826a5124" }, { "model": "openai/gpt-6-sol", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951710-0UYPeBZEIPCTExlUh8XT", "proof_sha256": "2c22543b414e389c34afb7b6706911d8be58b29eeeba2f9d3204546f0d01840d" }, { "model": "openai/gpt-6-sol", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951727-DRdQAeTyWvrC5A6YGHh4", "proof_sha256": "cfe02d9e26fcd92add1c149709f9d3cd42b90ce11a0186b2d0b7df9707e73de9" }, { "model": "openai/gpt-6-sol", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951741-U4lqOUIAvLsODIqRLlJe", "proof_sha256": "0bd4d70e3cf2d569da1cb605a630987b4d3903b3558f44189e52386bfaf1d050" }, { "model": "openai/gpt-6-sol", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951750-A9rMhaQWAIT9zu1fRS2m", "proof_sha256": "4b820c20543613b2812624e43bfe649eb9a10f11c6d8b28658e3833cb26743fa" }, { "model": "openai/gpt-6-sol", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951772-wJ0eFPr39MflYt05dpqA", "proof_sha256": "dcd5a03c6a808eaf83e9e70016bfebd1af158a69f9755be92683bc3f274810a0" }, { "model": "openai/gpt-6-sol", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951781-tX7pH0pA12mX2FqdXpa7", "proof_sha256": "18c35f8c45fcefe2704af471e0e3d0bcfd70ee22a773b4b365c737d815344b5a" }, { "model": "openai/gpt-6-sol", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951799-eG2gSnMGaSiAQN1Inp3a", "proof_sha256": "f9cdacf6e1cbb51e3a1c847d31e8378e855b1b15a459f462fef075208217a892" }, { "model": "openai/gpt-6-sol", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951810-vHizSeXIx5e4DhC819kr", "proof_sha256": "a1c6d0d0397752d85dd960c9fd8bcc30f2610aeb3ae36e7494421e28de0c0f73" }, { "model": "openai/gpt-6-sol", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951819-oeH7tMMSzAkionS3Crt3", "proof_sha256": "5c9ed4990dc29c24154ffa1f81292a58e319c666bcc9ccc0d328f1698f92f219" }, { "model": "openai/gpt-6-sol", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951836-9ZBFZF6Zm6Gi7PAr6kNE", "proof_sha256": "6c9e8d94a160a7cf59de444d2d3b7962b222d607f6ca256d6d55ef8be405744c" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950870-7T46FhzvrsQUFLaZlbSb", "proof_sha256": "8515b0818e494cc86b1a0a6db512a0cac726f24a527196dfa6c6f0b19b16bf9d" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950881-1xpvqz6GyOCbWo4R3cGE", "proof_sha256": "76bfb5904d4ac03517df6e883129b1a836b2e1d8c5d11c78954214e75acf85e3" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950895-aiDyH651cmCVeU3IMyTk", "proof_sha256": "17870f53f74e7a4c9302918f257462f40a42fde7960c9d8eac88f273fff6a954" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950905-zNBcN22m7FHnLlXbXmCG", "proof_sha256": "749ee0c833948bb9091bbc425ffe85ad8fdd7451168b2a2620ace0c71dc0482f" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951013-PzAW1tTwfehxCuZe0oe6", "proof_sha256": "e89631dec6a0f4c589d4f0cc18496f54e1580c6350547096daa2ed6e3143e42d" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951030-nTU6QxDMN6hwIa6aGueh", "proof_sha256": "b7e928249943d1db8586fe2e5a567d699b83d6e277bac47cebb8ad65b6f88701" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951042-ACsSKKqaRxggFZcrE1cu", "proof_sha256": "6ee847fc1d358e37bb33198bb45795116a2acdec35c4f20f9388823f04148d11" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951052-r0OM2P83Asw1mdddSXfd", "proof_sha256": "8e95a6e4b365c10a57cb3533ef4076ef054b56c131b9c46ad23c09e31905816e" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951064-hMO9IiOzfZQ8R3sDXnWv", "proof_sha256": "53fcde67a7c90d3e2d33b87d17a6f3b5d69f0c0565996a01d9f1a00df898c398" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951086-Ie9gUyWZAvdSnQUehuxE", "proof_sha256": "f90c1c74e5260f54ed0acf2328b10017913efd478062c701bd2e5178eed4941d" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951102-tSKlTH2ibADBFDfnaqQB", "proof_sha256": "3474f3b88766e9343d2942dddd10616f0938cbfb63b92bd8370cea5c46f6cd7c" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951112-bBHFjGhXZwzF7Ixr4J9Y", "proof_sha256": "d93ea331cf5687d3dcebb66d157d819c2685f527cd2b119880d90153e01defb0" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951131-UDutLMTeiFIBeXrVKtme", "proof_sha256": "b63d63fd06fde83a05f6b9047425304da5ba71fe81e408511c7075475833158c" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951141-czLmbXPhHEENQOkm6uUk", "proof_sha256": "f2b352739ded45d3c93c5ee59f7571ca74e4d47488443e38f2b67b041ff868ef" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951152-jnCXVmRypxAhzzqWPXaD", "proof_sha256": "dd360e9084309bcb7665d41d40a63176886586fc4fbcea35b5c6332dd8822d97" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951165-2mcGBWO0ymbYGAu8Uqq3", "proof_sha256": "ffe1a818768c34da274d0999f2d9920c1f21a2f78aa45c8245fe01d26ad0690c" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951185-wnBMDILt4DqMDzTULxdJ", "proof_sha256": "06fbe7d4de44968fa7ef16303206077500c65beee059df15ee4af361be82d04a" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951195-73058MqenKGBhQTIGyZT", "proof_sha256": "ea5c7565feb0fb2ad0052554d6237ae6338bc2113655be220dd2e265fcfd8402" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951206-yk7gBlkI1fe7oWcKp2OW", "proof_sha256": "8b9601edc460a8aea822936a7f91579ffe95e7fb941d66b8bde5acc4c16c5c58" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951227-7htaTPKYDLI7wajY3zXF", "proof_sha256": "4588586e4fa9d1aa504125e80ee30642a55b49b9a9b54fe6f57038d8ff306224" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951248-uXoAZEdtWKL7DeNKbAy5", "proof_sha256": "8729e2982e801f4cb3440354bba1f2a094f9529eb50425fda49bc5aa0f013d58" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951259-dU54jCZFYLNXpnloaytS", "proof_sha256": "292fc0054baac67fd7bf8ca9fae6759fba07d0e88bbc58fbf577685e31a1aee2" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951271-V3lH7HCpgY5fmqoG2s3K", "proof_sha256": "f8eba1069e882294bb8bc9f5a99cc261373435584306ef85f0cbe093d5111089" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951304-xRgzbFA5gwJkAyBUt2vt", "proof_sha256": "429cde4e2ae04fea22caba4800d1c537742225d980b978a2d3da9986457cebc1" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951324-klNs0SMzeOUmxBiHZa2d", "proof_sha256": "7fbcff791e637b2dc8198deaec0fe638b2285ab63767bee59a3b8ce5c33ed273" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951352-fqCA0YOc4cDRBu9rCFP9", "proof_sha256": "79492ce80c3e61f20d9800133cf516e08f037d31219ba0afca6024bb15e7069a" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951378-kTIWIZcOBgTB5bSraY9H", "proof_sha256": "f57b3581a0a847a7dbad73b00cd88b1963c0c5b57d2c8f23f50fd5ed58230b41" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951399-90TMJ4ilpua5qhT5qwk7", "proof_sha256": "03460efcfabe3c2800a833452e58af87bf7ab2a5dfb59b3e2b9806addd2759dd" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951424-ZY8WO10GVDUbdPp5N3K3", "proof_sha256": "34f7878225e3077cddbecc3c2be7cd0bfedad1af85c4416582ffa817e19381bc" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951451-VeZM99ewUMQOFho0wTyc", "proof_sha256": "71bb4251dc39919debac6899b11badf1d87d347ffa618eb3af9f63848aade1ae" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951474-awrdrcXbJ9SuEbjFMU0a", "proof_sha256": "f2a4dd5efc44f2ba3a98a5d8e2057df22465f1e60a031dd15a2cb1c3e45aca5b" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951497-uo5cLfCcif8UuEfGBByD", "proof_sha256": "079028372f1acee44071274e48a0a4aa2213acaae6e6f1299fb7929e273a6c56" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951522-ztoV16bOGZ7fdE818HXW", "proof_sha256": "a51c6f0aa99621e637343afae2aaa79413cd872099161985c32e5341a8e1c32b" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951534-T3wDbxOfQzTYVPXri9XE", "proof_sha256": "4328a377fac5467e53c651708cf395084d2bad0bb8a822299662fd6be9363566" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951549-rmpDtZ8tWxWf7Q6kwGBm", "proof_sha256": "13c8d9a2200b813f80dd6bde342289a3d7aa826c0c240c8b19e3a8fda0a57f48" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951560-UVa0ZbT50jekjwoDDzCX", "proof_sha256": "dc2cb5a099069db935e4dec578a3d6e7046f2ddaf099077140bdc9e2fabce997" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951580-6SEERNk34zZJxhCF8ql7", "proof_sha256": "a9425d4c270d68c867600a743321618d0f5c7f3768e7a92f4a159918912a3927" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951594-XS88aTh4Jg5An44Fu9og", "proof_sha256": "4d983ec2634fd3e3c8d5519ee9d88f3c602b7c06ca0b2275914cf6864a1af345" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951604-rClmX2GUPZdLrNaG2ZK7", "proof_sha256": "8fe8dacdc730fa7d02acca84e660e182b21273b55d504c40492afc184820c34d" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951616-QJf8wCVHwFidAmSowVoJ", "proof_sha256": "3bb5c93a6b327e29f33fc2334d8a6fee60ac02d869fe1c3715c984678c6fc02a" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951629-goj2pph6Dd0dYMKyPSwF", "proof_sha256": "6bc199a97458f6f9150c3d789cc2cd77bcd8033b66c93cd6b190fa75e680ecdb" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951650-0VVZFt6sNkAtkMYhW3bi", "proof_sha256": "6bbed33ec9c5e6a0820872bde25db0e99e4b5d9c2f901aae118b56fa66d023cc" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951670-6qi79rcU5wUFPVZlHhl9", "proof_sha256": "80f3de3262976fb676186462b0f9567c5a144524a9ea00373a5187854fabc894" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951682-PspJ6mvHsKz0cp2eCW5S", "proof_sha256": "7a2e6699d86136c9ecc5d7c0ec1d149dff3bedb8c7e657ea03ecbe2009c6b54e" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951703-l3O74bWBrs9bQdUuLzw5", "proof_sha256": "20a6a2502eb606f18e7279bd47440190c3766e820c842e0745ff67ea477aaf70" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951712-pg2kXujVLACUs9Jm0y6Y", "proof_sha256": "07b736d92e14243e1e68828028862e08a49c02e29b06d1c22146f72b160091b3" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951724-KOwwfF7kjJlpjyTNmtAI", "proof_sha256": "dbc9baf445d25228008e7407d942d3ba86f7b8c4383d6365cb387ee753fcd5c6" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951733-VNFeZ6uxdSQjO69qWYpm", "proof_sha256": "871cf0270defcdc7655cb59a1e51a46dd344f81af91ea6380124415d23e58955" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951748-HFQmdZeg8wGG54etsxvH", "proof_sha256": "7d8e715cefe46402c68c5ae907d371d71c3db0789de6d85e152847a2f7a8a590" }, { "model": "openai/gpt-6-luna", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951763-dkqgnMcEcHSDAMsSDfOJ", "proof_sha256": "1971258dc6bd4ab32489f58bdc757569df5fc0d6aab8bcb02f00b82b4cb293d4" }, { "model": "openai/gpt-6-luna", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951782-1DNW86NGi43vyUCxpCPi", "proof_sha256": "bc9c904785aa11f09e13ede83cc26719b3b36a4b5a00b6d7d09b6a1742588bf2" }, { "model": "openai/gpt-6-luna", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951798-bNu1oCuoFj3qqGIXNAa7", "proof_sha256": "1197137fc78c0ccf3f2a883be8c54642b3473e6c23a796504c64c3f68b0b46a6" }, { "model": "openai/gpt-6-luna", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951808-QH57pQdRLLY0MiWy3eW7", "proof_sha256": "d882fe39b3a77f3423890e4f80b536ba0de559445cd72ba6a3f3cc12e7c292af" }, { "model": "openai/gpt-6-luna", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951818-MMDicDky184JjU2N7AQK", "proof_sha256": "9a44f1ffc59c998d133e4bddcab57c467e002b4869643daa09083f53b0111fc0" }, { "model": "openai/gpt-6-luna", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951832-f0NhkJzfK9GjKxNf0kox", "proof_sha256": "cb472825702540dc23baee860deb93a123f22494a394a54ca4da4ea4712bd9d1" }, { "model": "openai/gpt-6-luna", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951851-WGFUedzsiUgx8S3HqTU8", "proof_sha256": "c10028101fe54c97d1a42a33051b2341725d082bb1391859e140d4951b9fd91a" }, { "model": "openai/gpt-6-luna", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951865-6GCc21OR74wf6cJ4oDJ0", "proof_sha256": "9b17b3da19c3135381fe1c7f358f99ab6c7fb6eed5786099ab48d2be6428d32e" }, { "model": "openai/gpt-6-luna", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951877-RhOdqWXGSvmF0Cuza8US", "proof_sha256": "d3199375d4250ec501cf1f44aa3f30171eab439d7c2dedaa1659f6196f53d9ab" }, { "model": "openai/gpt-6-luna", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951886-ribQMwmWdl4Igdq1dkzY", "proof_sha256": "f02c5105bc301fe4994b0ae75c2fd2d952f6b490fa0a810e0990170b6f6101a7" }, { "model": "openai/gpt-6-luna", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951906-YoOlXC23lhtMKnFQwvCl", "proof_sha256": "3309dfdcb32541f31d96df3fe2aa7b1479e254fed240f2a070aadf04a608e638" }, { "model": "openai/gpt-6-luna", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951925-b7BOLTHgq2NDAqqeB9bK", "proof_sha256": "d7d083e5e372fb825ed24d17fd61c1bb5949fe220358f5d330c3e13a836813bc" }, { "model": "openai/gpt-6-luna", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951934-1uUhYLgYO3to1RfIXgTs", "proof_sha256": "283b2425ae23ba5042914100cca7103a8f480b6033eb79beafa2ba08952af7d0" }, { "model": "openai/gpt-6-luna", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951942-Kk7SndwiHyMpqAbGyIAc", "proof_sha256": "9b8c892eb2df89b9e282d191c330ca71a655206c4d6d04d077089ea9352bc9f5" }, { "model": "openai/gpt-6-luna", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951952-Ka33uRVD466ySa1hDEfR", "proof_sha256": "0f6d7824acc32ba667350af56f99865a5f21dbcbf20e853d28d29514d54dbfb6" }, { "model": "openai/gpt-6-luna", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951961-ffBhK0HwTIi6TPkeOMIG", "proof_sha256": "3e9c772630caef9b08fee5ae64a938936dc4bf399eb2a43553959a5e0f99fdfc" }, { "model": "openai/gpt-6-luna", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951971-6W7sW8jPfFbExxeFiy78", "proof_sha256": "80fa7c52ea2e0389128c29190805a04b5df4d4fa5c5f408369f8e46e984560a8" }, { "model": "openai/gpt-6-luna", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951990-7OXv5TpygYMmB2PjZ8B5", "proof_sha256": "1b001fb1efac01c0f9907d3ceed5226f56bb30d54f78d730bcb79b3599406ee4" }, { "model": "openai/gpt-6-luna", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951999-eJGmNwngeXC15Jfensy0", "proof_sha256": "a392381f79f5f6e97aa479a7c60f909df44872f0b02f67b78885906500211c46" }, { "model": "openai/gpt-6-luna", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952012-mxbMKBVRkFZJDDrTEKLy", "proof_sha256": "193f6087f082d659296c4b0109196ff9b23c7857964ca45bfd9d350dc96609da" }, { "model": "openai/gpt-6-luna", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952021-zXcZ5EVLndbGxrsZrd2r", "proof_sha256": "e22ad064d36eb5431a19143d269d493a70a93ef4f83a50e1c90feafb458fef0e" }, { "model": "openai/gpt-6-luna", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952031-NxspRIGfBOr0TrdAGmHH", "proof_sha256": "86d51629cd9dda55b7176cb6afd711c75c2a76f0ee376afda12ca5450be233cf" }, { "model": "openai/gpt-6-luna", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952041-S7AKrGAD6Xir5FhkjLbc", "proof_sha256": "dde278a4243fa87b23a6a9bbab5f9dba8169d0ac57418305eab903620b2dd58b" }, { "model": "openai/gpt-6-luna", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952057-kbWgZTksa9ZynZujKexw", "proof_sha256": "29d78d959960254f11db230b17cb4c69c2ef436ed652d8ee28f4ec89f04997b7" }, { "model": "openai/gpt-6-luna", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952066-aKshxDm1Rbl2fhJrizQE", "proof_sha256": "8915d78c74f2cc37de8d18b51601f118d8995405256d4c546a3c05a7f3372c44" }, { "model": "openai/gpt-6-luna", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952077-O8tLlkR0gOprljUDgpgg", "proof_sha256": "aac86a1af03cb77adf2c1ce04d39b8908933b006c63a3f4c30b48502c81ce310" }, { "model": "openai/gpt-6-luna", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952096-A0KEK9NDwluAuYHAl2Pd", "proof_sha256": "f029b508b6e71835b18c479f10756e3e1f5893327d391e3cb97052ab81a34f0e" }, { "model": "openai/gpt-6-luna", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952115-f0q5ZvB7iYxWJo2gPYjB", "proof_sha256": "698e188faf152ed35ebdab5592deeaf79d2abf0deb226145cf8ed5f323e258b4" }, { "model": "openai/gpt-6-luna", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952126-p1Ya2imNiw8GC1F87OrV", "proof_sha256": "043484ca052b2f581000f9ed641c93178c7666bdff20b3fb8c39edc7b09bd5c6" }, { "model": "openai/gpt-6-luna", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952135-cfpRHWME6oLEovO0s9Zt", "proof_sha256": "b5db5e1d00a6a8ad2579eef3e2b7ed2b94d28e70009aef95506ddf3750577b47" }, { "model": "openai/gpt-6-luna", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "OpenAI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952144-Djs1bbMtQyTKgeoL8EFV", "proof_sha256": "2e72e82385c3fe2349baf434cd9fc700c786c124656631eb928aa9e42faca9e9" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261025T172733Z", "generation_id": "gen-1792951028-FdeKGGTlKKZH4KNuny9z", "proof_sha256": "5b6b23d65da7a8b8a8a565a6d23d72d2cbaba4ff1a959ecc0d747985260af53d" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Minimax", "run_id": "20261025T172733Z", "generation_id": "gen-1792951060-HqbqTOIDnQPTWsbIlcxZ", "proof_sha256": "6da66ed5cb4f8b57f76fbb594e18a3048f8a4b05f9dd1991a0cf0ec6d8702b37" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261025T172733Z", "generation_id": "gen-1792951087-7Xuc4OwX758qxb6LhiZf", "proof_sha256": "de9e442c306b1f08cf5d868ca10a80c7e6091386e29a65be7aa9b25b962c1fc6" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261025T172733Z", "generation_id": "gen-1792951100-7xnapEsbOMWAfE1yA9cO", "proof_sha256": "314f5baaf6d1f6a6d7ee624faa10a9b1cb029cc0fe3a1a70d69d06fa67184253" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "each term has degree $3986$. note $n(0), n(1), n(-1)\\", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "InferenceNet", "run_id": "20261025T172733Z", "generation_id": "gen-1792951163-KKPnv1tGUvv35VeI58D1", "proof_sha256": "6227151e95f0bd1487c188459e387db633373828c7c0d4c5186c316328b2bd3e" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Minimax", "run_id": "20261025T172733Z", "generation_id": "gen-1792951240-TWYIusUq0GPTRZapSa26", "proof_sha256": "25f4af8a509ecb3906771346a6a5dd26306fd1acb45fdeb53020e04bd902c743" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261025T172733Z", "generation_id": "gen-1792951260-4GZgBUCbYAAZZWHrKwOx", "proof_sha256": "f69df1c22bae0321784661a5cbe5551b041b4f9179a91b3fb8a1c8b59a09281f" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Together", "run_id": "20261025T172733Z", "generation_id": "gen-1792951286-eOgevczyKCaOx5ycm2Oh", "proof_sha256": "55bb0ca7a63174219cdf638972defad8a6d3bb1f5f40005a2ca0d11e6bde951d" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T172733Z", "generation_id": "gen-1792951307-iThYzlNL2ZVKUXHKjKLE", "proof_sha256": "78ae9011411904039a5d509936476843bb39341b290a5faf11f7b6f35323d7d2" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "ModelRun", "run_id": "20261025T172733Z", "generation_id": "gen-1792951325-tFRvtjEVPvsZLaVYjM6k", "proof_sha256": "91f9a0257968083ef34031939724b6e4351e5ac0f2f1a68699aae4aa0cdcc8c5" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T172733Z", "generation_id": "gen-1792951347-thtTtyFXxjohu1tFNPQw", "proof_sha256": "4698bc4e641f7d83ede2f3ede52ac70445b258f559c97b6c206f5f1cbe86c612" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792951357-X7sdhD9gZepwbeIlMBei", "proof_sha256": "a043211c92f537372ab486d00442c96918a1f79525b0f22d9144b2851b0ea869" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "GMICloud", "run_id": "20261025T172733Z", "generation_id": "gen-1792951381-ypEQVgIivDstZJ6GGS3x", "proof_sha256": "b3356333fbad13eaa78afa812b277020de6d8d69924c92cb53edbf74711e5ddf" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T172733Z", "generation_id": "gen-1792951400-72USkGV0GSeluxvmutIS", "proof_sha256": "a7673afd98152eef0ee2ceb9a777c71b43b4530128d4911a838bec1c7412762e" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T172733Z", "generation_id": "gen-1792951410-kEPFHcWL57AiNvCgYlo8", "proof_sha256": "015bfc9a9a66de7e91207b9cf74b7fe51270fbac5cc6738d852f58d8eb350a51" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261025T172733Z", "generation_id": "gen-1792951428-bv4aAC6bk18Ra2LUHrfY", "proof_sha256": "a5896848603b16955eea0b15b39062046fc2ec98b0d7a436493ba6b11ad2aa78" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261025T172733Z", "generation_id": "gen-1792951462-JbfTROp3AIrIHwCuHoQW", "proof_sha256": "69ff4d6369eb3a126d63672ba25aeea73ff9890479f5287dcbd0a94ab61d7416" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "GMICloud", "run_id": "20261025T172733Z", "generation_id": "gen-1792951480-paApFlVRdHrCGu9qMW9y", "proof_sha256": "1cbb9a0ed4244c95cf5fa755c532a1fb382c51e109b17eda166694e9f3a47af7" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T172733Z", "generation_id": "gen-1792951501-6ez2vMn5ffPf9bocQ21H", "proof_sha256": "c4d8986001369acd5b589cf3e6e44243ab30767adc89bec69bc992ef5fffd67a" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261025T172733Z", "generation_id": "gen-1792951519-dbLB1xTXpprL4LNz0k7C", "proof_sha256": "51d67a5e1656edeb48d5f7033e441f2773b3bcbf6f2ea2cb4b3fe2ab339e4464" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261025T172733Z", "generation_id": "gen-1792951531-MPNMgBZJbY4EZkp5kaTJ", "proof_sha256": "f3dd63bbb0671c48685c9bc9d541bc010935081208307109816c2b622e33325f" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Minimax", "run_id": "20261025T172733Z", "generation_id": "gen-1792951545-NupTpKSuOuxyd1pCb0qF", "proof_sha256": "9869ab952f1ed82c572a8f285e91e32ab1c57cc4cb4cfe0fdaedc986b7e4093b" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "InferenceNet", "run_id": "20261025T172733Z", "generation_id": "gen-1792951558-J3UYDZ3WFQMPRQgnJMsn", "proof_sha256": "6f6e9f5f83a199ef60c02b40c03c19eebd8e41b261e5c74cee4212ee3fc6c9a8" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261025T172733Z", "generation_id": "gen-1792951630-qhfjFNnOfsSNrDKkO99C", "proof_sha256": "f1d376fd026fc0a8aa20ab1515da734daa2de429b09fa3f5601de4a419853d0c" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Relace", "run_id": "20261025T172733Z", "generation_id": "gen-1792951660-UizJwG1f5vvrasKyzLek", "proof_sha256": "b99f287e3031c05ab72eb5f121630c896319fbd8f862856bd71d236361387cb3" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261025T172733Z", "generation_id": "gen-1792951680-CGkhesyXUDNgDh5cXr3C", "proof_sha256": "19f1bc532c1afd1758a9d2ab7d8d76fb6531e6fcbb45ab6be0a70904d6af3925" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792951786-r5V2Sg237dwCsldurlLs", "proof_sha256": "592181908a079116c5798e79def38043e4f01ff5042105e0c6f9feea36cf5efc" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T172733Z", "generation_id": "gen-1792951798-dlomth5ZsCNAyE9ZC3V6", "proof_sha256": "76404ccd65e190577b8a109fa9428a6723606f472a5d0ac2f68e5aad185361c3" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Nebius", "run_id": "20261025T172733Z", "generation_id": "gen-1792951808-GoOV5IkjR5kyoWmrN3oQ", "proof_sha256": "9669645117eb41cbb3c931f62a5923023ab69f57d55de921df3b83cadfb57784" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Minimax", "run_id": "20261025T172733Z", "generation_id": "gen-1792951850-tO9CaHDltmFNAKtkBS6E", "proof_sha256": "304f5939178dd94ef12684c5293acb7937caa0d5ab00c25cb2076a19b209590c" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "ModelRun", "run_id": "20261025T172733Z", "generation_id": "gen-1792951882-z7icHEdGare1Q9BBOWxs", "proof_sha256": "2340bdcf61d152e597e449ddd0b01390fb9436af0040124b83f0f0609d6a8ec6" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T172733Z", "generation_id": "gen-1792951913-COkRZ7DkjbGPENULGAsh", "proof_sha256": "e4fecf979e0325527a5ca42f7dbac58426866898ee038e4ab99e3c4f131dbffe" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792951935-Q9NHvVKHnMoKGSDAD9J8", "proof_sha256": "39fe9216ce9773f86f4ba1fc336281359f534fa9e886c86dfb1b43d40da8eb5c" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792951957-leMx2FZSf1XLXEiNMy5R", "proof_sha256": "8cc0d5c5a7af2f8c6808694203fe3f7c78c3fc74b0e63a4db635e342aa44763a" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "SambaNova", "run_id": "20261025T172733Z", "generation_id": "gen-1792951973-MkzFhc5p7ef7o1ADQQzz", "proof_sha256": "d0cb05c787ac901988ee1df9881b73b0c8db45dcb1fbd74e58e53415f60dc7fb" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Minimax", "run_id": "20261025T172733Z", "generation_id": "gen-1792951996-zF8dpJtH7CtSye1j88V8", "proof_sha256": "e93b06446f8f8a3b687ad8ca5ecbdc0899e20f3494d3c679de0238ab3fcd538a" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T172733Z", "generation_id": "gen-1792952006-PY7Eh4m3eEK2Ak4Il3wv", "proof_sha256": "cf11222572db0cbdefcbf320989f07a92f4dcb6656820e2390838b311bbce3cb" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261025T172733Z", "generation_id": "gen-1792952019-WjL21fys4mcNphcg015d", "proof_sha256": "9862b3e8c87046b0c03cce2f4925c7c4319ff408093a9e0516794e275abb274d" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261025T172733Z", "generation_id": "gen-1792952069-sx8tF5v1UCMm5MsCjuyf", "proof_sha256": "b174e5df0406cce71ca4ad98d3bab7cde042d3bd9580116c407dfd2263781a8a" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T172733Z", "generation_id": "gen-1792952100-WqltxxWUrLheymBIpxYH", "proof_sha256": "e45338a6bcbae8d3561b49401a3eb770199aa0f07444aa9649eccfa24cb68b0c" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261025T172733Z", "generation_id": "gen-1792952109-Rc6JikGxnpyd4TYsBsaW", "proof_sha256": "44fd7516dd5c94f40dd6f34ebea11f961679944f54a7c0d12232aaf82e1fd0c1" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261025T172733Z", "generation_id": "gen-1792952134-nXzuM4N0k8LBRn0PFUIp", "proof_sha256": "16fea43a618adf833ffe38be8fd564a65c745fff0c3d1e6a1c2fdacefadda11f" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261025T172733Z", "generation_id": "gen-1792952166-8z3t7sGLxVLTLlh7lAS0", "proof_sha256": "f5ec8882ea58a7fdfa065aeb31bb78ab3d6e6a86d6a313f4f145f319ffc2a33c" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T172733Z", "generation_id": "gen-1792952191-bP2WKnMx02L6PaCLl92e", "proof_sha256": "2f056e9336b2a04918511b661925aff59015f2dec2981746928592d8b147c1a5" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "GMICloud", "run_id": "20261025T172733Z", "generation_id": "gen-1792952200-WWfs5rrTgZ8VSUuGEVv0", "proof_sha256": "144af6dc8bab23617c55ee8851348ce111d92eb45865940a11f1ba8016eb9f0e" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261025T172733Z", "generation_id": "gen-1792952208-vNN0AQ3SoOvqbV8tSmKU", "proof_sha256": "42586ca320bb37d8e9b9308da9119b31944ef6c09aef94d39454cd451d3e1d33" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261025T172733Z", "generation_id": "gen-1792952233-fxhcEo04CF9vEZxpksKq", "proof_sha256": "e4849f17867c0dfb4222af0dfb9c2679afce7d8d0bd2159fd1a7646167a83a2e" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Together", "run_id": "20261025T172733Z", "generation_id": "gen-1792952261-XU3PMeZjQOqokTWdq47Y", "proof_sha256": "f8ba1072e071459e80bf532a06e3eeefe6f2907076af1fc13a1fd79cadcdd882" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Relace", "run_id": "20261025T172733Z", "generation_id": "gen-1792952282-mSI2jgEiisdzqN9LFZnU", "proof_sha256": "5dd3991c78b4fcf83e9c7919db1195771d06fd3b2b837469b362f77d79c25b36" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792952311-7kWjIiAWmSyDMOYRhpA1", "proof_sha256": "3485e17d6194991de968ce44397fcff4ccd32b6c360423e93a298351b2b47143" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "GMICloud", "run_id": "20261025T172733Z", "generation_id": "gen-1792952320-ONRT8UlS0lEbvt4qr8Y9", "proof_sha256": "80f75cf490522fe04f2d9a6ff02b7729a1279c24b2d273f5ed0fe4e33337eebe" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261025T172733Z", "generation_id": "gen-1792952358-GAx2X9XQzE28FIHadfmR", "proof_sha256": "9cf4261963aedfce1f21d99a2994f5950b8fa29b002692486823799e172a9c8d" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "ModelRun", "run_id": "20261025T172733Z", "generation_id": "gen-1792952390-XGPzpoG44Ij69Ehx7N2y", "proof_sha256": "0d78e076d87cab3bd31d9f6c8c897940378dfdd588619de8a89f063c77e367b8" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T172733Z", "generation_id": "gen-1792952407-DrgFENEt0RyjFlL6f0dG", "proof_sha256": "beaff307564725e56e0f9a89dc36bcedf36c4ed9dfff4f427a681805b328655c" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "ModelRun", "run_id": "20261025T172733Z", "generation_id": "gen-1792952422-a7K1fY7wC2dms2ukwFHO", "proof_sha256": "d4dc72b3536fc533572faaea4ae16c5692735fbb588b02f0fe3a58ff50557487" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T172733Z", "generation_id": "gen-1792952430-gDE5fiQaxUuifcjt9UAK", "proof_sha256": "60320ebaf0e4e52861693f25c4e52f80c8e2adcef1b1cd9256a82811e11977f9" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Minimax", "run_id": "20261025T172733Z", "generation_id": "gen-1792952438-LaIfDtTxLZSiI18o4gki", "proof_sha256": "3aab53c759ce7dce6ebc5b5037b837fde0c23fc67c9c413e13b771639ebd353e" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T172733Z", "generation_id": "gen-1792952447-jNT05p4lTAWWwpH66jAi", "proof_sha256": "6b87f975a482b366388041615e9be0dceeed0a0e87f1257f9984f60da9478ca2" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Minimax", "run_id": "20261025T172733Z", "generation_id": "gen-1792952456-i6jHFC4kzWWcKohKdvFU", "proof_sha256": "dad1baa10c9e7c8e61886b62cfdc8a5bfe41314aa1914804a251ddfee461e6d0" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T172733Z", "generation_id": "gen-1792952468-qDqxrIBiDndYa5oGMda4", "proof_sha256": "15aae128bba60c591e29083e777fac26aa75329daf57050841fca8489c6547cf" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Relace", "run_id": "20261025T172733Z", "generation_id": "gen-1792952477-ODzGePq5sNPMdhQ3lPJD", "proof_sha256": "ae9e831ef31c420d16588920e10d2a789b32ef973d7d32809a69bc9964a77631" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792952485-HXHwUNfiNxP6CB9z4LLG", "proof_sha256": "10345e0608666efd09f5bf39b50618067bd188c2a6bc7ad97eb3dd81ea8043cf" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261025T172733Z", "generation_id": "gen-1792952493-NF7AUfuQv82fZv2Dlk2K", "proof_sha256": "5897f228b8e45b817d6b348b085385ca75c7b6ff020126f2c1751f83197cb4a3" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261025T172733Z", "generation_id": "gen-1792952517-yhyA0Y2I3SgmRnffuIRb", "proof_sha256": "30c3c2ae637cefb44e273b3945273f60d2abe518df361cbf5ab7d63c23ba244c" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261025T172733Z", "generation_id": "gen-1792952535-mLRJkWKcvycmP5cOtpIj", "proof_sha256": "bca6395532a539b62c53d3ded040468d6e34260691cd121056e81a3c3a88758a" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "GMICloud", "run_id": "20261025T172733Z", "generation_id": "gen-1792952545-dhYFSyzlc3sruUwzFUN3", "proof_sha256": "b1a4201d258e495b76b4be0801879925431f4b8054ddbf5a86c0c23e49c1d20e" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261025T172733Z", "generation_id": "gen-1792952553-VfeuW1o96NhWCDzVjRkM", "proof_sha256": "2e54cf1568e88f6e176b4060bf77a911544bc538da49b4755549a8a637c74642" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792952564-bB23LVIQNoT3CU6e7234", "proof_sha256": "a569c3b14b63ac62bcd62fe554d84b17d6ec93d74f0d0eba3f18161f10bd5d2c" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261025T172733Z", "generation_id": "gen-1792952574-acas6P0NAGQK56tJKg47", "proof_sha256": "32af394a1beb9cd79a59e57d42f199b36d6ee951a5f8b16c02744cc6d4b43d03" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261025T172733Z", "generation_id": "gen-1792952584-3XJak1qUYKUnbcaCMK0A", "proof_sha256": "3f4dbdd2e003572413366417b2dbbccd6948265c1073f6693cf933a2609ac5c1" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261025T172733Z", "generation_id": "gen-1792952605-3xSKqXYwLWtI3zpQzrkT", "proof_sha256": "b70312762730692602d1a5dc8f1d1d66d72ea4e628c9c622b2b863a78f7bf78d" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261025T172733Z", "generation_id": "gen-1792952615-wL0aba2jfn5a1Odowq9X", "proof_sha256": "84816ca8c7c58e40befc133b68f7d680aca8f0df55b7d32d965048f016a429d3" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792952665-vGojuzZdb8jV87DqBShY", "proof_sha256": "0bdec7abd1d99da4f7f386e01b44f62818738b509c68e6099139b608ee73fe3b" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Minimax", "run_id": "20261025T172733Z", "generation_id": "gen-1792952682-b3hyXM1Aqe9UdxZCTEiL", "proof_sha256": "2f6b6baffa1cbd845926cd9361de3d7b542e4cf3fe8d705e80107dacdfd50182" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261025T172733Z", "generation_id": "gen-1792952710-yQV9MVC40wkCJ3fsekUw", "proof_sha256": "1dcd9c1675a3f3dd352290dc4f288712aed757a4f264b9de664ca742a3ab3b3f" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "InferenceNet", "run_id": "20261025T172733Z", "generation_id": "gen-1792952719-BUbDLmHE1sBDnXGeUjqG", "proof_sha256": "9f81f1aa19b89af64870e73bc1f5b8969f65de4b7d1248b278801c8d03cdc285" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "SambaNova", "run_id": "20261025T172733Z", "generation_id": "gen-1792952740-3nlkmYEnom6R2YML93Jl", "proof_sha256": "b89319c5b6c70eb079d72c1fe112d5dfe7042414a7d149f98450aae1575bec24" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T172733Z", "generation_id": "gen-1792952760-Kq1Abz5OQmUmDBVZG788", "proof_sha256": "71323b04906afeedb61ed56cf39412e99e93cc0cac8c9375b0958d786d0bf1c2" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Wafer", "run_id": "20261025T172733Z", "generation_id": "gen-1792952784-6CKdtIgkOPjPcIPKVzkO", "proof_sha256": "5aeb38b32b3adc4a8f5f7797a0541d6c26d9bb8ad4ad3c28efda3b47a6e6eb81" }, { "model": "minimax/minimax-m3.1-flash", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Together", "run_id": "20261025T172733Z", "generation_id": "gen-1792952814-xwKGgWPBtMvkTOoWtbVZ", "proof_sha256": "8cec56a2627197dd9c03ce1e45ec29ecd5df58e4b4af4b957ee2f41ae70c236d" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Xiaomi", "run_id": "20261025T172733Z", "generation_id": "gen-1792951165-gcKPXPyny3QgPzdmyYzs", "proof_sha256": "e704989b8e29fadfad257df88c91ab996a2ab235280850f0449f4c7e3734f194" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Xiaomi", "run_id": "20261025T172733Z", "generation_id": "gen-1792951220-DDv6Q6ZyTNr88FY9Jalx", "proof_sha256": "99eaef7b506be59416011b546d2d6e5c95bc5b6e3e6448bfaeec953cb5367acf" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T172733Z", "generation_id": "gen-1792951356-pUyk2SyWDIx6kIqIDQqh", "proof_sha256": "49f8e16bd3fe83de982fd416ddc5aba492cc68d1420acb2d09457101b16abc79" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Xiaomi", "run_id": "20261025T172733Z", "generation_id": "gen-1792951388-27c32JQtlJ5cgcu83myE", "proof_sha256": "e5b968ec17c8bb4ff4211c1ee7ba9c70c537fa2f7a3ee7daae8fb66f301bbc33" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Novita", "run_id": "20261025T172733Z", "generation_id": "gen-1792951480-py0JeZ5BJTA6smOMUAa2", "proof_sha256": "b50075e2ec503fb7f413387de782bad2870e64f1617089f0d1506a05c5e5899a" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "GMICloud", "run_id": "20261025T172733Z", "generation_id": "gen-1792951562-yPY1b7Oz6h8HD1Rw2jjH", "proof_sha256": "7db161238903985e1f01ef84c742fad3b5b7477da0f02cde376191bf14cd0cc3" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Xiaomi", "run_id": "20261025T172733Z", "generation_id": "gen-1792951586-QwI97wTrhRsEN2dBHo2s", "proof_sha256": "8bc85416c643847bb582c863dd26ec34707f19fadeb73c4e8196ace88b4d3f4a" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T172733Z", "generation_id": "gen-1792951617-jlv0kiGEcCfP0q6Wb5MQ", "proof_sha256": "eeb67a3aee95826606efb250d0244886ba654dffe51bf2a9af072f6367a48e2d" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261025T172733Z", "generation_id": "gen-1792951709-QDF7PQaiVfk5YDfS247p", "proof_sha256": "b2e1816926df3dcb36beed1b57b5a80074a66d7487d62f6a887c3ab5f733b362" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "GMICloud", "run_id": "20261025T172733Z", "generation_id": "gen-1792951733-jtzuqcZdXoRLcSDqmyzY", "proof_sha256": "488aad3418e3e90e6dbbf901ab6de18772632e944bb110583b5672193b923919" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261025T172733Z", "generation_id": "gen-1792951803-R2bknFvEteTfCH4Cr1o3", "proof_sha256": "aad943dc5b61e73b2b69dee4885abb5d2e8e33bc73a9e8e438d381007b2868f9" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Xiaomi", "run_id": "20261025T172733Z", "generation_id": "gen-1792951830-6gW2h4pba8occW7FVhIt", "proof_sha256": "96ebdbac0343c461f2d1036565eddf17a89ecad5e3f66759373459fd5fed05c8" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "GMICloud", "run_id": "20261025T172733Z", "generation_id": "gen-1792951968-Dva35UMtYAmRsDnjBMhT", "proof_sha256": "036065399435b07387ac6ba4d9dffe12ebcf99433f6f83b132e7e409fdf0de23" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T172733Z", "generation_id": "gen-1792951988-qRmUOgNt1s3arw7qkbMw", "proof_sha256": "7718916730a4caa68bfaafd92170e996395d456daca0486caf4f65e5c49e765d" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "and $x = 5$ gives $p = 87360 = 2^6\\cdot 3\\cdot 5\\cdot 7\\cdot 13$, which is not divisible by $2^7 = 128$, nor by 25, nor by 9. so no larger divisor from the list works (1920, 1200, 1440, 1680, 1920 all exceed 960 or require extra factors; 11", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "DeepInfra", "run_id": "20261025T172733Z", "generation_id": "gen-1792952010-gHxbziFiheygkBxcW1SY", "proof_sha256": "a120f4adb0281d6ff1439b47a0613b1b9e9117637f15af1d9b9f6f56c3c44b23" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Xiaomi", "run_id": "20261025T172733Z", "generation_id": "gen-1792952052-BxwVP4ofUSNPX9GS3dIn", "proof_sha256": "cc6532db7d1f3679c4c91d45b26dc47cabcd7b8c7c111d1f2825ac9370fffed2" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T172733Z", "generation_id": "gen-1792952102-GOfZfas4NkHmlPlrnImA", "proof_sha256": "102edf29114501cac9d0f5fae1b41d3673deafe77c0f0cf9b3bb818d4dd06e09" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261025T172733Z", "generation_id": "gen-1792952141-14je6s7R4oHnJL1mcbyJ", "proof_sha256": "68261727f1479fde051904ba60a567ae5c6a518c0ebb7aae47ba196684f84c16" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "GMICloud", "run_id": "20261025T172733Z", "generation_id": "gen-1792952173-ZcmVIjWO5n1hOHyFnBXV", "proof_sha256": "130863ba0508842fd379ca4aeb76ea8cd3cdfc9c33c890a0910de89fd7902b1a" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261025T172733Z", "generation_id": "gen-1792952201-6gTRmBfMV8uEAqLiJ5VE", "proof_sha256": "ffd7b7bd3dfdcab7d9df1cd8592c9dffaa9544588d695c757a984889898134f5" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Xiaomi", "run_id": "20261025T172733Z", "generation_id": "gen-1792952233-fXM58uCth8IVqgytcDtP", "proof_sha256": "1f4200338b2a1419df77d24aae2bc783d5e28e1b4efe5a6148f2ad070dde9a8a" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261025T172733Z", "generation_id": "gen-1792952272-oQb3Hxv2itqi4WQhsFoG", "proof_sha256": "c254e757e78e3e966183db21542283f18c6d348b00c9e7221e7200d20c799997" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "StreamLake", "run_id": "20261025T172733Z", "generation_id": "gen-1792952324-SIq96ZWZcCq9Trbhs94b", "proof_sha256": "182335af4bca23016a239be5caff0b7712449a83a721027734c6422651a690cd" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Xiaomi", "run_id": "20261025T172733Z", "generation_id": "gen-1792952397-yvf9eSzLjXJ4QAPJjm43", "proof_sha256": "4749ccf77d6bdd743a1cecee78e05cbac5d088d7b6b95da76b0c45fbe30c1bac" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T172733Z", "generation_id": "gen-1792952432-PRMPAv7e1bkTf5JxOLZl", "proof_sha256": "9bb3420deeb3e253b6e38241eae08aca15918a39ea3704cbc698ae9c182390fc" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T172733Z", "generation_id": "gen-1792952477-1yGLkPikqvFLlAIj0Zg9", "proof_sha256": "da995a1cce3306b954f8af9c0bfa2623dc4229493fa159fa3174f3266178127a" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Xiaomi", "run_id": "20261025T172733Z", "generation_id": "gen-1792952563-mla4QxxZhwpeKOpVArjj", "proof_sha256": "d44042d120ea5f355fa5cd337ff54989f822b694a3c2d2fe06f6559e5399c024" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "GMICloud", "run_id": "20261025T172733Z", "generation_id": "gen-1792952630-MpDvIsJoICockb5fm5gw", "proof_sha256": "7e4307e6803d5e4cef0427df96e54cbe34fb1fbf6e501109cb592f61ff3e74ea" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Xiaomi", "run_id": "20261025T172733Z", "generation_id": "gen-1792952703-khf81OyVsIG2vFZGrt9j", "proof_sha256": "67f42eb57f4c62750249745b2772b24583a478ed7adbd2730bd41bde309565dd" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T172733Z", "generation_id": "gen-1792952839-wrXTRIx5d76ohgOaJWBl", "proof_sha256": "064e2bc938e0a86d52eb4eb934069f9b8d5b1fb714f4a4ca824725282abc04c4" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Xiaomi", "run_id": "20261025T172733Z", "generation_id": "gen-1792952866-381hwy6FTNuS13U1Sp9T", "proof_sha256": "be0e942dd632090990c8ef468a35e32829a8b9446035ec950eb7dd347703c5a4" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "b (80%)", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "GMICloud", "run_id": "20261025T172733Z", "generation_id": "gen-1792952983-o6Wj59a6p2k6Qekwss5u", "proof_sha256": "1f5e0d6fb366c81342972e6c82c8250a60c73447edec37b7afb6dc5582e65d70" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "$$r = \\frac{4.18\\times10^{25}}{31\\ \\mathrm{d}}", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Novita", "run_id": "20261025T172733Z", "generation_id": "gen-1792953057-0ZBfiSN8ICyFtqIQBs9p", "proof_sha256": "b8a94fef1099c8e61b9111d7af9db10a7558669f91642a0b51194a5cf47b58cd" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Xiaomi", "run_id": "20261025T172733Z", "generation_id": "gen-1792953130-pkfexE02YvScIoxIR6nG", "proof_sha256": "5caf15015c09c5ba27c2efdd015a8672a9a5e8dd5014fa510bbe5777cf403968" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "GMICloud", "run_id": "20261025T172733Z", "generation_id": "gen-1792953251-mEm8iwWkYVliat9Gmaqn", "proof_sha256": "a0f2ef5952f28b951dbe3b020bc90338232c4eb515f1d6c8d508e0db93253937" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Xiaomi", "run_id": "20261025T172733Z", "generation_id": "gen-1792953282-KucRqWVVqDVDcUw6znVO", "proof_sha256": "bed40e090a024c389d76101a4dcab5ccbf77405ff96e8f9ab9a57f864d4282e2" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T172733Z", "generation_id": "gen-1792953339-LPrmGSQGWkIQnYSvd94U", "proof_sha256": "2bc6fb94674c1471f9f7cbd4782fe5ea53b3d57e691ad79b28a8b4aa57a92349" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Novita", "run_id": "20261025T172733Z", "generation_id": "gen-1792953395-1r83kH5Q4csbCuiDT1LW", "proof_sha256": "56816998882491e37c96ec71211caae6a5977aecba439319ab2495cdb265ff38" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "GMICloud", "run_id": "20261025T172733Z", "generation_id": "gen-1792953494-4GgTiUPFZscKewg9Zjhm", "proof_sha256": "45d16b670726df7def35bf97f7d6f453835e86c1a6b03e3a045a40f54242acd8" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T172733Z", "generation_id": "gen-1792953527-wr9cvPNLIvdz5h6CRdfj", "proof_sha256": "59e52795385dadef210bde232b05b31e788e3a98b909aca39168ebc8a8df1ed9" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T172733Z", "generation_id": "gen-1792953649-3hRMvs0pN31zJVzqcJqK", "proof_sha256": "0b1f95f154b5fdeaef53970e071a3881f032b992a4da5a1e4421e251c5804cd1" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Xiaomi", "run_id": "20261025T172733Z", "generation_id": "gen-1792953687-avjPqJxJBJYmbKv1a0fe", "proof_sha256": "9ccfb4edd5f3636f9332dfe43e825132f8bdcbadaab52990a466a6f2a92bf188" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Xiaomi", "run_id": "20261025T172733Z", "generation_id": "gen-1792953746-yxsrA1YJDxcUK94akJ85", "proof_sha256": "1a3a0f8023f4a67a7a269128fbf9de6afda3370c876d58e6e00ddfff55303991" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "GMICloud", "run_id": "20261025T172733Z", "generation_id": "gen-1792953896-HgqUO7yx9n7dSGX3BXL2", "proof_sha256": "68d1f8becb934e91b7c18aa779ade257eb80c1b5a0002239105c42a56391a99b" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "GMICloud", "run_id": "20261025T172733Z", "generation_id": "gen-1792953938-zemgenzDhUV9mALPfwL1", "proof_sha256": "56f4bc03eb75866653c34cd1899179a60aba7c8677eb1558a05c5d1fd042aa12" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T172733Z", "generation_id": "gen-1792953950-o5Sg81xMkNFHAHYiZWhh", "proof_sha256": "d526a23d76784e54462558c96492032ee543d23b07729c31895a15fb6ae00ee4" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "GMICloud", "run_id": "20261025T172733Z", "generation_id": "gen-1792954025-eg8b0mAqjZjxU3R78ZIY", "proof_sha256": "bd33365161f02ed0a50da6759ccdb98024cd2cdc067fd6d490bbbb1f09d26324" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T172733Z", "generation_id": "gen-1792954078-NepTkbVdNda5hq1NI0WA", "proof_sha256": "8f58494e6e4be167a0065c50414e07a0e9d7fda2e6ef38ef0ad7e8decbcdab56" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Xiaomi", "run_id": "20261025T172733Z", "generation_id": "gen-1792954132-sy1WZWWSRernNe2HkqfF", "proof_sha256": "791b5376e941ef755a9b08f6b7763e1f01a2a05a404d28690487d23eb5c74e29" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T172733Z", "generation_id": "gen-1792954264-Ido9y86d9eumVRcpHz2w", "proof_sha256": "49e15085880c4fd97e6d14a65a4cc08982e9aeb18686c63892318e5e4c6178a7" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261025T172733Z", "generation_id": "gen-1792954299-Qhmhytou7ECkpgmAhEDP", "proof_sha256": "6a30c8d71fd22b969787348c58b75b2c03837d27d41c9ab9130b567d04e9639e" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Xiaomi", "run_id": "20261025T172733Z", "generation_id": "gen-1792954384-S0RqXA3rbvvW2xn9ZiIk", "proof_sha256": "a3b5599a5b58271f2b50a3ae1fcf83fe2408b1efdd7ea6c842d6ccdd225b3348" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Xiaomi", "run_id": "20261025T172733Z", "generation_id": "gen-1792954490-UBLLVsHwPvssDZ8RxEqR", "proof_sha256": "61a75dc0ff6ff585088d6744c1c63e012d40f843cb212e7fb3b975a0acba8efe" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T172733Z", "generation_id": "gen-1792954561-x0UDl3BWckXUxX95X5b8", "proof_sha256": "3b6a4d8cc0f6aac4cd4e5057ef5ecb31b9bdee5e6a29ba8f07c28a0df4af1014" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261025T172733Z", "generation_id": "gen-1792954606-BfYNd32Wn9LGRUMuY0j6", "proof_sha256": "975f2f07f93b6b9e523a18b19364f1aa131a60cee548a727058cf6e172ef315c" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T172733Z", "generation_id": "gen-1792954698-O7TsA1Of4kggIQbKtybf", "proof_sha256": "1c6b8a38862d153d9800b73d56c887a0be39aad5410cdc26357b41cb7e7b1bb6" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261025T172733Z", "generation_id": "gen-1792954749-It1MSSU0S0vICWnMEm7M", "proof_sha256": "2fe7abacdcb517bdb64c87039d48d20f71764859654e547e8c8f6d5f076d9d18" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "GMICloud", "run_id": "20261025T172733Z", "generation_id": "gen-1792954820-PgcJhaqOtsLeuRcWkiHe", "proof_sha256": "2333c8d2660b056eb7b6bd8d24a762978da1a9db7a34b8b8c1756956e51ff943" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T172733Z", "generation_id": "gen-1792954834-q2JHr6YEm20hQdRVLq5q", "proof_sha256": "c5c211a3b45ccb4419eb6c5bd4dcd5dba6fb5e942f11d4301cc91950afc4fc81" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T172733Z", "generation_id": "gen-1792954856-CIXEOKOkKrbklKsvWkhY", "proof_sha256": "b9f745d57d21b61e4ed9a2767e6528a01b0696e71144570f114dbb966d32f057" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "GMICloud", "run_id": "20261025T172733Z", "generation_id": "gen-1792954869-nuMdEPBhFRzA1h6ITzSH", "proof_sha256": "6a6cd58f0439b5cdbc1814591f3888bda0db4c2801bba9b95a8ebf3577ef7fd1" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "GMICloud", "run_id": "20261025T172733Z", "generation_id": "gen-1792954886-j7JKa1DhUbumj4paTTfe", "proof_sha256": "d3a15447f819a9bc63fa511f0475b9af694ec40430d2df68358961e5cdde577e" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T172733Z", "generation_id": "gen-1792954900-WZtH4D1KIWQox1CdeHKv", "proof_sha256": "a4550e2771c1e44bb545916aedfc9460e2890b2b721bfca24727aeb7566e9594" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T172733Z", "generation_id": "gen-1792954920-ty5APeebVeclfUeJl3d6", "proof_sha256": "0bd03a538b6255d35af4854a637714cd167d71b1ae791b235d32f3037615c0f9" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T172733Z", "generation_id": "gen-1792954938-ORveLoEOBQ6AGYiNYkv6", "proof_sha256": "7076f4aee6eeed8e9528f5ce052db82054ce9732f5cec1b054393a6bb3ae8980" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "GMICloud", "run_id": "20261025T172733Z", "generation_id": "gen-1792954950-MszYkJ6wZsxSLHWwLZx6", "proof_sha256": "faf62a45b17178fb10a44f9cf346bf1a696a7ef36b58d453cfebc2e934b00fd5" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Xiaomi", "run_id": "20261025T172733Z", "generation_id": "gen-1792954961-8GiSRVjL37Ar1kTszvYm", "proof_sha256": "cd91562f7aabed4b3b87958a6ad85df3996a1ca3956d9d49632ef823ef9567de" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261025T172733Z", "generation_id": "gen-1792954992-P9fpaDa661tJ3Rso2s3b", "proof_sha256": "c019fd2a7cd28cc65cad6564f7b3502030d308dc308218f5cb2b41821263807b" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T172733Z", "generation_id": "gen-1792955043-1gx6mJ4hYKLKHIIbIXIE", "proof_sha256": "a33c12b3d9afbace96d9c1c3805bb05c17820fe562090bf33d49ed10f1fdff4d" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T172733Z", "generation_id": "gen-1792955079-7W4TlPMhxfPAdrLlKQ2i", "proof_sha256": "9156da45491dcfbc0c435eed8ab61508c53890d5c0658f0ab6cd5940e9f5dfef" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "GMICloud", "run_id": "20261025T172733Z", "generation_id": "gen-1792955095-OhRLylhUvj8z7gxtmbu5", "proof_sha256": "9a366ce04cb56dfc1545e6c748b1552f4411652ad0a176ed9ccf2e8bed07b351" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261025T172733Z", "generation_id": "gen-1792955108-nczhv2bjB6NqId2cNSl7", "proof_sha256": "9e48911c7ac12a0d4f9b8003776f6fb496c8c87c73c63521732db57a1a4167ca" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Xiaomi", "run_id": "20261025T172733Z", "generation_id": "gen-1792955169-0jSs1lV7eTQK2RVie07l", "proof_sha256": "810e2a4f95eae03f4717373e53fc9587720c037eaf5a446e5149f4adff2c5242" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T172733Z", "generation_id": "gen-1792955186-HdRg1LsF2G16LDDng1ST", "proof_sha256": "5221fcf65d85f767f9f3487839bfd0964f3d763d3e2196b8c54daec8f48bda00" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261025T172733Z", "generation_id": "gen-1792955210-jOsZ6HFzbXNGC0kFVMcy", "proof_sha256": "53dbb07c97b5cb140fc863ad4d3ea097e1741c97955889a8d6debe6825d04d4a" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Xiaomi", "run_id": "20261025T172733Z", "generation_id": "gen-1792955238-eWOVVaIabCD9FNUoyRFz", "proof_sha256": "00d85ed16844b48085b0d0f28afded2b158def6185ae8a132b059438f1e27a53" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T172733Z", "generation_id": "gen-1792955283-MvlifC25tI8mBAS3UyUB", "proof_sha256": "94e13dfa757f8b7036178fa5e162f5f7f82c3c481f24f9e56ea5bcee7fb634b3" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T172733Z", "generation_id": "gen-1792955389-bcjLDo7S0EdmbS2KkyLv", "proof_sha256": "afcb262843e6c962c423e4bd5316e100a483563b12d1b60aa9460e60fb191354" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Xiaomi", "run_id": "20261025T172733Z", "generation_id": "gen-1792955438-03RzC56F1aB6JWHSV2R7", "proof_sha256": "03c61bfec81038891598dc3206d177d06740839de4cd7bc632dc25f2cb3ba087" }, { "model": "xiaomi/mimo-v2.6-pro", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261025T172733Z", "generation_id": "gen-1792955491-0YLi0AykHV9y8ZPdN6ZT", "proof_sha256": "95552532ce1d0ff88e9c6e183a4f3fd70de0323de1e3180b3518b8fec2aaf395" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T172733Z", "generation_id": "gen-1792951203-fUs7kDuHWkknzIx83HSV", "proof_sha256": "bdd90cd0d0afcdcb768b293f52f0dca6136a4bfa81bd091cafc63e6a9757a2a5" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T172733Z", "generation_id": "gen-1792951227-WniR5ytZBV32jYkkMo4y", "proof_sha256": "94ed64519b15e6b96482b56bfc2b6858aed5eda184ea5dd1af8748b865fd9d88" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DekaLLM", "run_id": "20261025T172733Z", "generation_id": "gen-1792951249-UMftyxkiJ04fJC3Detc0", "proof_sha256": "0ae8185a608c15f3d80e10c5b3ae8eb1e57cd299bb01dce0c52b196bed6ae687" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261025T172733Z", "generation_id": "gen-1792951268-5Z0XYOGZltbOrcXN3Wu0", "proof_sha256": "21f18cef84757f78fa6ffccc5a6ff1bd87d5a26e89869baf3ae75610aecf53fc" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792951459-ThdMf57psEpOtqpVo9tF", "proof_sha256": "e221fdabbb46be113458e8600fd3142a091debfa4a8d72c6eb35bfe77dcf5ef7" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T172733Z", "generation_id": "gen-1792951484-69Rq8ZZyUZGqwLYU0TLY", "proof_sha256": "45a16e8a79822fccfcc299b913ce8ecad06194c3a21e13f14cf2e1617a17fa26" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T172733Z", "generation_id": "gen-1792951508-kOmYAJI3BRv2cCe57U66", "proof_sha256": "8510af73e10badcc89e1c2e61151af56b5de5fb32a4fac6f2b7829286c8c3dda" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "answer: c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Near AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951532-AXLhoA6yLy002Zb1KE18", "proof_sha256": "678e781bdaecd6f872e408d59fc3f19b103b84f54eb61f126da60b20e38b1bbd" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261025T172733Z", "generation_id": "gen-1792951550-ToYRDq1teJAQwAkI0l4a", "proof_sha256": "a2a6e0e84ae5d46fd7243efdc1cb4ad5ec7c11d39a502249b9dd12c2444181a1" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T172733Z", "generation_id": "gen-1792951630-6ZV6xhS6LmvQ4fi0u4FB", "proof_sha256": "eefcef34703c9df4397029a48a6dbda71abe6f1994b120934a346f85a15d0168" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mancer 2", "run_id": "20261025T172733Z", "generation_id": "gen-1792951644-VObUOdlwBg2ibUciTqzm", "proof_sha256": "6a0b65a2d54b17608d6562f0386b660f9091dbccf3588e238d75f49099b6f3e3" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Near AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951673-fYyQGZzig4CFhdj8uzZv", "proof_sha256": "188c89afc03e74e4ddf57a2b5d5f6ddffdcbc45b813525797b63121004adb17f" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T172733Z", "generation_id": "gen-1792951685-5DveJ4L4UouJgvi48Tjw", "proof_sha256": "39781b2a99a54b2e1c9a80b540bcf9bfe39feb85979d82e301971ea5d7b2aa56" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T172733Z", "generation_id": "gen-1792951695-TuiniI97AfoJNOxLThV3", "proof_sha256": "e7232bf452c23cd18055cb3ca7ee6ec383320f9d1ce7688d665555f95fe8bf90" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Near AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951715-4LerD6QzQZg6ArsUwPNQ", "proof_sha256": "c5f552b161c7cb88b81d10745a81e8747ebdc1149be6892949deb4e0ef3c805e" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261025T172733Z", "generation_id": "gen-1792951738-3YmHSURKZruXyaJ4ANBV", "proof_sha256": "e317c3a28a97861518827fd502c14c7d73a3bdefaabac6d4259fd3d72853ef9d" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792951752-DlpYWYJldKC8bfsKGchq", "proof_sha256": "51c897f976f8ebc3a3af588672aa88b708f20eb5a3f8a900be73593b29d83951" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T172733Z", "generation_id": "gen-1792951763-nYagOVz9035XY5biTIpU", "proof_sha256": "b9c97af0f7d63c08bf125b6175093d992583229458f2286332238c63f1b81f91" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T172733Z", "generation_id": "gen-1792951787-TM4R2lKeOrBSUI5Vovsf", "proof_sha256": "7d68c4ccbb233816b942cc9336b6d04490ab1856fa2b76635064f51b589923de" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Near AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951803-O6QKFF9maPo3C7cifxr4", "proof_sha256": "f172618e7315d6a00cc4738d296d549d48a48b137ec9be503a5a780d6cac0fe4" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DekaLLM", "run_id": "20261025T172733Z", "generation_id": "gen-1792951817-ZSJd2eNTDbA1CZcxmiy4", "proof_sha256": "2ddaf5495519f367955c7388c51aef85f3b160a6cc26d9e203ca2e4932c207d4" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T172733Z", "generation_id": "gen-1792951843-VW2prK5fvknb7j7qLtEJ", "proof_sha256": "ee67c4814064dfc2e036c7271ea5bffb4b3c7d667e7fed54afd1d6a19aea1d33" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mancer 2", "run_id": "20261025T172733Z", "generation_id": "gen-1792951876-tE4XWUiiQkds6ZI1v06Y", "proof_sha256": "09e6b263340df9e1550105da195b314f1892d6a871062c50fb2eae22e5d1dbf5" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261025T172733Z", "generation_id": "gen-1792951885-Bo7eCjazInkFoCVLgaYF", "proof_sha256": "daec1dbf708c5274aeb96f7e08d090535cdfd8d7b0589f14ac9213776bb6510a" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T172733Z", "generation_id": "gen-1792951903-GVU1nOQSi4ob8yslBV0p", "proof_sha256": "3351b2ffcaf1f9dd4fe892e77eca3b0fcc0ccc559478d53f662ecf10187b0ef9" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261025T172733Z", "generation_id": "gen-1792951914-ceDcUqnmFGDWnt1p8Ncw", "proof_sha256": "fb0546c99d4a11edc5dcf91b0496d618717e1649b86226f5aaaf52565c083cc0" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Novita", "run_id": "20261025T172733Z", "generation_id": "gen-1792951946-KArxXdaz2kLRhTkqgjOV", "proof_sha256": "1111751133af8586eea98208332ba2caa376e4001a345544108265febe41b201" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261025T172733Z", "generation_id": "gen-1792951969-Jy1fjDEfRx6lC5wnoth0", "proof_sha256": "df75f1ca9218bfea631aca60c27b289fc636f9771d6c578697cb2fe265c8f65f" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "h. none of these", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DekaLLM", "run_id": "20261025T172733Z", "generation_id": "gen-1792952003-x666vrsxQZW8RzPE22vk", "proof_sha256": "239f704b719de9cb4b489a750d3fb789e3df08c1f218b57d69273c2c4b89883c" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "\\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Near AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952056-wK5YUqj7yCkPkmKq3miL", "proof_sha256": "648cd59bab811f268ee68571cbf094e0734008cb9dc4ace00a7d8a346b183268" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792952077-QImpDYqIk5ijFUaB0FrD", "proof_sha256": "3d1fdcbf3d9825f36942671cb1970f59151e71f938203d325766caca9072b516" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Near AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952098-y7rFT8NsoHw1rfzf1bmI", "proof_sha256": "f8ab1eda255a6c2db94eae73aa2849b8fae7568e0c6725fd0dd4f6e98dcfc4bf" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "\\]", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T172733Z", "generation_id": "gen-1792952144-6jyzyDKYMDoczNNGT5IE", "proof_sha256": "e2b2f8dc1a8c3fc6521e81320c8973929a48b3ad300c9d291cc2cc90e75613c8" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mancer 2", "run_id": "20261025T172733Z", "generation_id": "gen-1792952162-Fmcgk8UHzoCXynclqFN0", "proof_sha256": "ec85808cc221a01f29458d0a688a2101011a9e62afb162bbff92c361523d794d" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T172733Z", "generation_id": "gen-1792952184-ZJP2heVIbhgcsx3Mdmba", "proof_sha256": "92129e5e92812d88793f97d150f46391843d25b3ed27f880c98fcf694bf4bf44" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792952197-GdpLiSnc1zzBYrMpTuiP", "proof_sha256": "55b73a73840ff36bab6ddf81f84b877cfe3f81bb69d69f63b1c5ed8864e25d88" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261025T172733Z", "generation_id": "gen-1792952225-j6kY0LoOO9NoAQZyNqCV", "proof_sha256": "f8c372bc7c11356fd1881175ffe21062cfe6471370a7fe5ee6f49e2a7f26671f" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T172733Z", "generation_id": "gen-1792952254-1UdJWc3FFOmDhS13QNL0", "proof_sha256": "72e026ce12650189d997e6be07eb7687449eb07a50880fe5263d1b6f0544af48" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T172733Z", "generation_id": "gen-1792952272-11PkIJAH7RcvpGjH1OGF", "proof_sha256": "353bc961ba6ac48dddea66ff58ecb0995209e6f147976fb42438615b22de4f57" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T172733Z", "generation_id": "gen-1792952280-c6smAfDkKlBdqZNNLNsz", "proof_sha256": "e1c83a9cf8e53f60775253dde6a4ab06844f7a844ddf213fae84431519d774f8" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T172733Z", "generation_id": "gen-1792952289-HsZtzJsA7WGW2Yvwqrua", "proof_sha256": "8a3cf1fd6705740f35b1d6f9cd1c6ec6276b40aa515d0b6d5d48555b83b6ce23" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T172733Z", "generation_id": "gen-1792952297-44SF6P4t71EEom3D5u1U", "proof_sha256": "022bc175f1336b5793b7ac38f1f3a38ff51520f9cdfea4c6dc7cc72a63e0d0aa" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Near AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952315-svY2Hph2X39EN3eGP6NF", "proof_sha256": "e66c976ef40598c07fc022ffe9f273eb8d9a3e45745ab236b323618500e35f68" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T172733Z", "generation_id": "gen-1792952326-1zrX2O78biAfq0eJFh6K", "proof_sha256": "6918d5df423f43440ce69b956ef6358ab4f4c94d1ee8375038067013655a4c58" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261025T172733Z", "generation_id": "gen-1792952351-ATJ6xJV1wZpetfj8IAxK", "proof_sha256": "98dc54e0e952a2e61ea196c19541f9dd1cf17efaaa30577cb9063389c0445917" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T172733Z", "generation_id": "gen-1792952391-tSWu3fcY4gugp0CZpvwE", "proof_sha256": "3a1dbfd9bb0d9ec832be26f7dfd70f4a9f263f593c7e342a53ab1fb45c69d1f8" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Near AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952400-tWtfH3naGEizoDE6HwPg", "proof_sha256": "79e7b444b5af88582557bfb09ad65ecd4d55ef32a92f56706b03970749e1972d" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T172733Z", "generation_id": "gen-1792952422-DdGQ33EmEMZGJq7HypzV", "proof_sha256": "8d296687a172cb0a6d925a0855e7dc4180abdcc12d8c3bf4244a0c8bb5948aaa" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Near AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952453-LK3kEa1CSeRfVfDyE1fQ", "proof_sha256": "25aa757c78d18c1eb40b928f09ef66dd2cd5db345a4408c02a9f84c6e79290e5" }, { "model": "openai/gpt-oss-120b", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T172733Z", "generation_id": "gen-1792952468-aTNZ2kqkfQ4V19EUBpea", "proof_sha256": "d4fa72c0bd0013163c041f472d5362434227f229b7373a93a126d3a14ec19ed5" }, { "model": "openai/gpt-oss-120b", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261025T172733Z", "generation_id": "gen-1792952476-AJ1bZyjhyVIIYWiVms2o", "proof_sha256": "add2b22f041a06b6b682413876a3c58554ad149c983f6bf32cd544887a76db57" }, { "model": "openai/gpt-oss-120b", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "BaseTen", "run_id": "20261025T172733Z", "generation_id": "gen-1792952509-casd10JmnNSS9naQhEKi", "proof_sha256": "fe4def02d089ce86cffbda52ce5b4fbff173852a72045ed62dbe8ad0eb11d6e2" }, { "model": "openai/gpt-oss-120b", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Mancer 2", "run_id": "20261025T172733Z", "generation_id": "gen-1792952526-rhpjsdQ9GetvxgxcA2A4", "proof_sha256": "69d1115e6b9fcc7ec954ab820b3dc14ab31b5b4618ce0a7ed62b464e095e4fd0" }, { "model": "openai/gpt-oss-120b", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T172733Z", "generation_id": "gen-1792952542-8EuYQ7orBjQGsEsGAwTV", "proof_sha256": "2b2153eeddadfeea4c759ce2a2ae56f95bc1bf5f1e19b0095579ba27ee572848" }, { "model": "openai/gpt-oss-120b", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Crusoe", "run_id": "20261025T172733Z", "generation_id": "gen-1792952560-O7aJtYTPHjQgML1nxibK", "proof_sha256": "44b0888afddb4bf62c29861d1692d424f3656806ea16c14622f1db0b44252f22" }, { "model": "openai/gpt-oss-120b", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "BaseTen", "run_id": "20261025T172733Z", "generation_id": "gen-1792952567-x25dIJDndqvjEC8Wee32", "proof_sha256": "40d0d9eb7fd141f686bcae0f6b7230b846a6b88a3d10eea1e5d0af4fd6f66544" }, { "model": "openai/gpt-oss-120b", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261025T172733Z", "generation_id": "gen-1792952575-jCLT9j14IDnPivgzFPvh", "proof_sha256": "416e32d2579ef0ab08a03e4bd31561da5aa6d20bd441978278222f6513b85149" }, { "model": "openai/gpt-oss-120b", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261025T172733Z", "generation_id": "gen-1792952598-k22bXZQx9l8hwNCX9ogi", "proof_sha256": "58699015a3f92976031154ac35b69e6c997a67559259116d84543c32271ec231" }, { "model": "openai/gpt-oss-120b", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261025T172733Z", "generation_id": "gen-1792952611-DDJqwJmrF66X50wqFU98", "proof_sha256": "792e1f9472db2d24d0b5e47126450f61215457eb08d302fc387a9546fbeeb882" }, { "model": "openai/gpt-oss-120b", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DekaLLM", "run_id": "20261025T172733Z", "generation_id": "gen-1792952624-ShTD3gTdYNNmjavFUJRl", "proof_sha256": "f678e8f0b41b3ac012044c9d9d58e315eff156a92d68958e10fe9de595216842" }, { "model": "openai/gpt-oss-120b", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Venice", "run_id": "20261025T172733Z", "generation_id": "gen-1792952641-kakumAUtmNKEpg5nYZq6", "proof_sha256": "d7181b512860d5e06faaaebcbeac2dee918f1d11b3b52a0049ad7385066082c3" }, { "model": "openai/gpt-oss-120b", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DekaLLM", "run_id": "20261025T172733Z", "generation_id": "gen-1792952663-si6YegA60V0kxGNYoONa", "proof_sha256": "fb10a1bb5ab342ae1b52c892df06fdb3c6c773c49a281f18e0f59ec01097acf6" }, { "model": "openai/gpt-oss-120b", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Phala", "run_id": "20261025T172733Z", "generation_id": "gen-1792952681-ua7eE32jBCqY378XGFbu", "proof_sha256": "f3ef46cbddd79739a0b3cb6a89a43e08ea584121de929f5a7685c9dbef46027f" }, { "model": "openai/gpt-oss-120b", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DekaLLM", "run_id": "20261025T172733Z", "generation_id": "gen-1792952699-mKSP1SO5qrLBsqdhFG1t", "proof_sha256": "3a41ca1983f8326b6134d058ab1fb59f6321b538cd753083ff7299e534828e6e" }, { "model": "openai/gpt-oss-120b", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T172733Z", "generation_id": "gen-1792952711-DLLWXZ3HpgMfOfOnnGZI", "proof_sha256": "218d54dc0890f919dd833119ab26e10c30a7fab7844a55f1a46591e30741ab4f" }, { "model": "openai/gpt-oss-120b", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Near AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952729-sXovB9ZRx45l5ToSjree", "proof_sha256": "b084b6c0c612a832edaaa2d60a03d324f07084950b158cbce811c3ea6e9b18e0" }, { "model": "openai/gpt-oss-120b", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Near AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952746-FLAJJBJXPNxhlLSBhrfj", "proof_sha256": "05b7b4dfd26d18039021c7e23718031071c162c309798663da0e323318159ec6" }, { "model": "openai/gpt-oss-120b", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T172733Z", "generation_id": "gen-1792952764-JkdcIXtbjp3BtpMzGgfe", "proof_sha256": "d471ce8a808bb41935952f8d4a34a7ecd63d505c327caa0e69370fb3efa5a2cf" }, { "model": "openai/gpt-oss-120b", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Crusoe", "run_id": "20261025T172733Z", "generation_id": "gen-1792952781-dJT8ebZtcwfhQV13A8Ko", "proof_sha256": "4059051cf9d9a02f127be7ca99459982fdd44d9ea59e9d7e811762aeb67244e7" }, { "model": "openai/gpt-oss-120b", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Crusoe", "run_id": "20261025T172733Z", "generation_id": "gen-1792952820-EPt1k43OZbThCOvLGcFm", "proof_sha256": "9883a481aab74d23bea9ecc1195b0ca83a2f2169c67eae0c629262d8a7c3f6cb" }, { "model": "openai/gpt-oss-120b", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T172733Z", "generation_id": "gen-1792952842-8KLKoyiQ868Qd0Ji8s1R", "proof_sha256": "79d3f0b6c6a2e84ac7d908411ef50af8e14fbbf1b41853690cb46fbfad00ad73" }, { "model": "openai/gpt-oss-120b", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "CoreWeave", "run_id": "20261025T172733Z", "generation_id": "gen-1792952850-JFUM72JpUnV0kLEewJOI", "proof_sha256": "acf2214803c55cf02e5b57a8b12a96e0214a501a537a6cca8eae156f61f56300" }, { "model": "openai/gpt-oss-120b", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261025T172733Z", "generation_id": "gen-1792952882-qoJU6P6JhpULrIlcJMzp", "proof_sha256": "d660a385f842ee32466af4a01f2a87c5a20e7cfbefef3500913d50a970a6722c" }, { "model": "openai/gpt-oss-120b", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DeepInfra", "run_id": "20261025T172733Z", "generation_id": "gen-1792952889-NVjMB7hEqo1N5ypOdjBJ", "proof_sha256": "9b5f1e2b02f5aeda5f1b9ed75c4467cd9f25783e6f863391708fc2be190c58e8" }, { "model": "openai/gpt-oss-120b", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DekaLLM", "run_id": "20261025T172733Z", "generation_id": "gen-1792952926-ODDzcHrgKRdev6FtVs4N", "proof_sha256": "1dc8d2dec93638c7cffe04580fbadd4e6357a73997280452ef4e62d9d2fc5052" }, { "model": "openai/gpt-oss-120b", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T172733Z", "generation_id": "gen-1792952947-AkfHezNiko27EF2Hhe5u", "proof_sha256": "77f57792bb07ce212914c1276a03f6a7f0f8d68c792f1dab6d566d7b9e4c5e59" }, { "model": "openai/gpt-oss-120b", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Crusoe", "run_id": "20261025T172733Z", "generation_id": "gen-1792952965-55yA8Uv5Uuenzp4hMGSF", "proof_sha256": "7c7f754758a9c62ce4a9ab231266be5cbc5434d45d85ba4477a11ff8ed3dd5fe" }, { "model": "openai/gpt-oss-120b", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Near AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952982-bM25F9ADjkwhv0beMkqh", "proof_sha256": "eaeaca936dd481e35509d3b613c9888ef1487ec56e5b7c3eb3d4986c0af8e624" }, { "model": "openai/gpt-oss-120b", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Nebius", "run_id": "20261025T172733Z", "generation_id": "gen-1792952993-aD6SE0cyTU1feDRLjaVM", "proof_sha256": "83a150d05c3f2f1c074abc09a2dc233f58032884d25ecf03d29e86b777a15a59" }, { "model": "openai/gpt-oss-120b", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "AkashML", "run_id": "20261025T172733Z", "generation_id": "gen-1792953000-pJ32H8IhXCz7gj8JyoNn", "proof_sha256": "ef62b5473e81c7abd9a0adb45068108258714b461ead7cd248a5d9e46b8af377" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792951846-RiNCBfHilPvSG1mhL9OT", "proof_sha256": "12f727e7cc1ae0c3617772689a7985f4662373c2f1734b638681ad2dd8a25348" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792951918-NZoJO26aWjjBmrNyCb4l", "proof_sha256": "1e2dc17e2638c45884645a8898a6e217d75dd9bbfb0cb3e81db222aff06fbaaa" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792951990-2zj7ERuKPG1o6r9fMi3w", "proof_sha256": "0632055b83509f5fb472eb30b88779f8509482141f14f3292dc8d4e1cb5f0ecc" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792952049-eZlEokXgowz4tb3WPy05", "proof_sha256": "5ffe28d93f24dc47adfc8aacd5af3973dd4b4fbccea65f24ea318bba0933d5ae" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "the final answer is: $\\boxed{c}$", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792952133-uNuWfemmHyAW1qbm0L12", "proof_sha256": "30dd00ce9bbf04e6e5c5677f808f336d4e6d387e047344099b2ec97d5c9b76d2" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792952280-cs5gEcJ90lG1ZRVfaJJI", "proof_sha256": "15772c3b9d2f834d5072639871ab2e3f2c9400ba6f902d2dd5eeddb27f739f44" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792952340-E9RqHfcIe0ux1nyOE4A2", "proof_sha256": "0990a7124efb272d43fac1518f1c5518a4f566a7f0b10951f6abac30398979c6" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792952384-Yr9IuzonnqxbWC2qvwct", "proof_sha256": "c96a3d9af1b23e5519ea01e46fc2efdd1aa8c59aa47071fe27cc2b7b5430a14d" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792952439-GzzF1LzhIjwauUHDcrHx", "proof_sha256": "98af21fdabdf23a6ddf2c57bd1fb13637ae749aaa7b6af56349480319f679d6b" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792952502-2KVMuFSVYMCkEfctRJNh", "proof_sha256": "f8d7598ca28b0ed2917738060b70a808f74c282971c80634dcce5b7c1717d630" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792952556-BGtJwCIArmyrwQ2V4Zkd", "proof_sha256": "a0d7a0617ff324a52b3b231009cde701a8c1dd5189d28e447139883e6f34f082" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792952597-TVxlzTMdMW10yYCpum94", "proof_sha256": "197c719dffbcdeebe7918ebc3a8fb668db816cafd7545a209390ecc5d03bfc8e" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792952663-3tQq5blwVK7lTJEXSmve", "proof_sha256": "b0573263c2bfc3e19ecf75a3219f595e4da2d4af7f77bce915e00f7323696ca0" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792952703-h7NmbTNQk3jic9P9kSmW", "proof_sha256": "a3355cb2461d7335cb79590a4f3e0681d506b84b5453cf7dd023a3a50cc25789" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792952816-GQgNaKsSITVIy0dWjgul", "proof_sha256": "9aaa6c8b09e739ba04ac90437e2651bc0c18be7cf4ccfd493da8c216d7b64339" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792952880-2IqsDL7RJFnSTypX3vZ1", "proof_sha256": "a3444139299a7be6f2fb38bcdb5b37eed543bf5f769c9f1aecb51ed218b64b0d" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792952989-s2Q7eJOJzW91pP3b8Bhp", "proof_sha256": "bcf4b6ca65f71e02fa45c611227b8d9be0b02a24f1bdfb40cfc811e1ebfb6fbf" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792953060-9qHq6JsPXP5b53HpeSy4", "proof_sha256": "45fbb86e3651a7a8a3de985e9b689ab48373f590280511dda348f24549943de5" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792953147-Rrm585uXITaEhaaHg4Wz", "proof_sha256": "0663962e8ccde5c4309e2b22a95a6fa71bc4c5a93721223490a14978a2979af0" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792953226-l7Q9gBup7SSnrZm1R3af", "proof_sha256": "e74d288243ef44a0878e55de9cf833987108a3d88697aa8a770e53f18ac6e1ab" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792953279-AJBKz8mOogV3GdAooUhQ", "proof_sha256": "ef4e357663c62bf13d63fbb7f7bf4759518bd01ca64edb4be96914bb46ee3afe" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792953329-L2jQgISZnX5ua1BoKcFS", "proof_sha256": "9cd216ad15d9eb8f88dc395413469e613796035d803b1203902f9e0a2f5f1141" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792953406-4NWc3S9m3fALRP2dnDDh", "proof_sha256": "4e1447832eb5dfdc0e07fc10e7c9bb795613ba4ef36adab851d4d5b146297a05" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792953478-0tlip6EDKWKycgcbWDOs", "proof_sha256": "8c874c0563c1738f4278e97bbfab540da040ec9758d21d7d4e47652c186f773e" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792953532-dGLvLVGjX6zt8ya4NAIv", "proof_sha256": "16b2e077f4a61d102a3b3c50eed946bcfeda79f6b5abd01b5635ff7fbd1a9274" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792953601-nP13cnHGMBVzrlC4rNE1", "proof_sha256": "5e32e0b82604ba1876481d6a5def4195f3ef4ddc9921e4cf18bbcdc04ff1a5ea" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792953648-4WGdo5WbqFfwOaSi3HNz", "proof_sha256": "386b175eccb59b37f8cee4e27410ec9c6b38c5e6d4ab526c640f80fc02abf791" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792953734-rGGWPuisGeMClH4uUTRH", "proof_sha256": "f20d0c5336112388575b1359651d15f5dc02b75b846cebe366c625d542f9ff8f" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792953805-NlJhX5Pl7aJNjq4dRZRI", "proof_sha256": "accc6062ba8ff1ae8985e3420f53db94d61294bc0226aff5ca737f136dd668e9" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792953895-rlXaspQNpC8gjgVgRHM1", "proof_sha256": "df2ef10a9ff806c4cfb913e6e08253b91931849ef41a307767d5602002818047" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792953964-auRmTHlrxuJ0jvDwLWCm", "proof_sha256": "04c4f7d5aee6d1fdff9aac99cc549941a647051a4d8796fab90f1e736ea77753" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792954066-BRNqMV5N09m2P7T3Mk8I", "proof_sha256": "42d4a4545bb93a747629127e2a2eb62847b7344292b9884f7abadb9f83c2dac5" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792954156-w9Wso6TtrjTb0Va9Bsqh", "proof_sha256": "81d4e914ed6f4e5cb4056eb6f53a03ed3a97c4f550ca7867a161abe4c933fc69" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792954281-F25eVoFUU1eGctjzB1Re", "proof_sha256": "a93d71f664847c9c3ecb61ea2e4a86405cf5fbe50c11b2c4881813cf05d0586a" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792954365-rc7FqNpiiYJPdOXNllXG", "proof_sha256": "92c29b71fac43ebd544f5acc2f4ed997bc7323f6091eb875aa5cb62ed5d548f8" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792954431-gzmEzP51hRAVFHyNroLQ", "proof_sha256": "d17afb7c32d7e291167584d48c0aa4f1ed4b0c2c4d0ec38578e9c592f4f99fe5" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792954515-zjeEaeEM7b6YFxTu8yca", "proof_sha256": "85eb90ee3f2214800b8f76fc6d6661478f091dcf42a9327bc92bada2ede7eec3" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792954574-aurmlGvoNYlcOHOxxNk4", "proof_sha256": "77b06814a69414ce6da48593c1c81837594fef0156dce96b897cc05dc34c4e58" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792954642-0lrJpqpK7mrQlACsaFdd", "proof_sha256": "d2e9028bd001c76c1a020bae31a50a2bdb91ff277b4ac98b1cb1372442a36726" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "the best answer is d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792954700-HVdvtRpvfmkmY5XkzIGj", "proof_sha256": "8379e75f3ad083116ab4f23386f72122dccf195f9cc5de249adf12ae058f2fbf" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792954760-yNhwxwCCQ8OUk1l96RDx", "proof_sha256": "ffc538c5c288ecaf5b8bfe747380b481119ea6e08fb5a4d73b3d85f8bd337fab" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T172733Z", "generation_id": "gen-1792954825-K5NWzeBE8YcnmczhVAHd", "proof_sha256": "73f98be7c5bf02f4be82adcd1b304107598885962861ab538884ca1fe019ce95" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": null, "correct": false, "errored": true, "refused": false, "excluded_not_in_manifest": false, "finish_reason": null, "host": null, "run_id": "20261025T172733Z", "generation_id": null, "proof_sha256": "f02f932a007c68c557d5617da407274d1f8f7b9402f8297979982641fc03b2ac" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T172733Z", "generation_id": "gen-1792954867-fcMWAZGAxD8k8r2NRVhr", "proof_sha256": "ae5342724cbabb294a90cee2d4c2d45d297cec56982965caa23b5861c00d5c4d" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792954908-XuBSpQPuwHwJug5ZPO8X", "proof_sha256": "f67b118448e5a8db4c93e60cdd328c6724385b69924bca838177380fba6e7ce5" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792954958-oDfm6gekdTevRN3XSeu2", "proof_sha256": "15cde1deab0ea7011c03ce4d6fc42ab3e7c1a20d0648bd19b0d99d783263600a" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792955140-7TXeJXl6uoNOu8hdTrDJ", "proof_sha256": "ac33614ea0a3567b7490029d216c70397ee7bf483ff700e305e042fed3f4c976" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792955237-hRom8ugPlhCAW3YRoHa9", "proof_sha256": "f1724e2832ce605927a952ef34b423a6702b642c19029fac382f5f492571fd61" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792955338-T5rd1Dp79zHnnilmi1fi", "proof_sha256": "e2fe01e9edd7aa8fdb31d7d891ced774c964d6659df98ac5cc81647bf016b58f" }, { "model": "meta-llama/llama-4-maverick", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792955482-ubQquYG8Ku5nLPlAtJcQ", "proof_sha256": "f060b089f4e80611fddd53065ee35be10d198518752e63d0b265579bf109c159" }, { "model": "meta-llama/llama-4-maverick", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T172733Z", "generation_id": "gen-1792955599-0xVpcSRjAHCUD1EzsCFS", "proof_sha256": "fddb7c24748424478f1ab6d06be57f247ceacfec80c2b071a747a6b5203c539c" }, { "model": "meta-llama/llama-4-maverick", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792955633-GUJFwdTYCVIpr0Vk0sW3", "proof_sha256": "bc9236d81a23a966260cd39c90f457d72f0928d5d5d6eb38d26b413c5a06ecee" }, { "model": "meta-llama/llama-4-maverick", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792955738-ICs2x7PSGX0Zdb0faYM9", "proof_sha256": "6b78d866b3153dd14ae19533fc733735b684886564e7ebcb2e7b117ec585aa59" }, { "model": "meta-llama/llama-4-maverick", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792955862-HdbamkAvXsy3a6rB9oRR", "proof_sha256": "40615554397c0178f8ff9d1317c7a7d8e65d3d92402f3d626518788141e9181f" }, { "model": "meta-llama/llama-4-maverick", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792955976-IgsXs7aXfL2f05DnF0hN", "proof_sha256": "07310d361981a55921d4906cf99025fed80fcf3f314407da9e01d7d468686959" }, { "model": "meta-llama/llama-4-maverick", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792956069-uecjPRlj3XBvbzqMYc1S", "proof_sha256": "242de09b14fd525a7801dfbd26a40375c30e518ee1313ade741180f8170108f3" }, { "model": "meta-llama/llama-4-maverick", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792956157-4trQxTYJci3RAiyGXdMd", "proof_sha256": "b9964a3cc71e72df313b462d950ce8475f429f7f034136348dfc2a14737425d4" }, { "model": "meta-llama/llama-4-maverick", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792956258-G9VRnrrxDSjWjXlk64hg", "proof_sha256": "5ec2ead09c1a4c9178a16eb991190925aeb6242d03234e9387f50f4f6767d238" }, { "model": "meta-llama/llama-4-maverick", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792956357-xSmSswLzeUg61CtIHgZE", "proof_sha256": "b5eff53cfdee61b2f23ec252fd4122dcb45d23c4306e74717ce4b5577ec766cb" }, { "model": "meta-llama/llama-4-maverick", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792956450-e8NN6wNJRdKdIOWnS2fU", "proof_sha256": "c90bad575ceebcc11e0a7156d288a068c86ee500e8ebf4492a70f7e5facaafbc" }, { "model": "meta-llama/llama-4-maverick", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792956524-CUHZ8oMYSXw43GrUpSnl", "proof_sha256": "3be5ffd907252c927f9389819601437effc471930aebb9b30b1d68e6a3803172" }, { "model": "meta-llama/llama-4-maverick", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792956614-0LoGN9MxL5hXIjrjI7SK", "proof_sha256": "fd6c3783ef7ddf79bb2af3be78d812c80e931d14ce52686fdf14d3a9b361cbc4" }, { "model": "meta-llama/llama-4-maverick", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792956703-iLaFDlTVBj1syooYGpyZ", "proof_sha256": "5511a0cbc9478ef5ac4e498aed1093d4e51bf6a28670266a90ffd517dca13438" }, { "model": "meta-llama/llama-4-maverick", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792956780-ay1sV56UrOx2Ci3HlxDG", "proof_sha256": "c31dbd1b2b9f90d432b7c644040e006f292b45bc03f783a140c22fce946c8d8d" }, { "model": "meta-llama/llama-4-maverick", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T172733Z", "generation_id": "gen-1792956845-Owsca0Hy4ZCUbLBmUt7D", "proof_sha256": "e8e815ab7cd2bb1f159afb64165cf10db4d72401bc7be8a39f7259ee18096a96" }, { "model": "meta-llama/llama-4-maverick", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792956857-MZMQjZS30g4l2D0uNOpm", "proof_sha256": "1cd869d4bc725b137bf6be6f28eb73138ec8682e1614ac75084d0c1f3c95adea" }, { "model": "meta-llama/llama-4-maverick", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792956898-IN61COS10m7yBmeXHQYa", "proof_sha256": "be2581b3d803fd418add7692a4d76add5dac95c3c06317f8397fd242784deca8" }, { "model": "meta-llama/llama-4-maverick", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792956958-pJ4VZOXZ2Iv3a4TbJxV7", "proof_sha256": "efe242abf7d35f6ec4033ca0c395eac5789361ac64519a3cc1764b611891d128" }, { "model": "meta-llama/llama-4-maverick", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792957032-jOOLtHrA77liagRFRlLR", "proof_sha256": "b67fbeff73f8bf1c0dcf797949866dccf1ecf3bce6d6fb4bb7df6d164c722922" }, { "model": "meta-llama/llama-4-maverick", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792957072-K1Lo3J97Ex7rz3o2fMHh", "proof_sha256": "fe3e4866ff58d168db616634b6c08cc8812cd0520294b57a79840e2b734a2f7e" }, { "model": "meta-llama/llama-4-maverick", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792957148-nB14D9Srjnep1Z2FJeoL", "proof_sha256": "c4840054606acb7726ed4ab6c053298c8c9f268c1d7d94e70c9bc113d68d416a" }, { "model": "meta-llama/llama-4-maverick", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792957213-nyzejUAT4CH4uYBJHTc4", "proof_sha256": "7519bf8ee9ab894cb5bd28f35472cedc585780a17f23443e740cc0ea2cd79d2d" }, { "model": "meta-llama/llama-4-maverick", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T172733Z", "generation_id": "gen-1792957360-7sdihXQVcVm2cfdGq8lm", "proof_sha256": "a2ce38ffcef545930533bbc75f4d3dac4296340d39c07269a2f5af7326aed3ad" }, { "model": "meta-llama/llama-4-maverick", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792957374-LJU5v4B9zLOK7LipHR0j", "proof_sha256": "b5d63f09f1911366b657461e8639f10796479289d3cebaca5828f8d101a7e048" }, { "model": "meta-llama/llama-4-maverick", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Parasail", "run_id": "20261025T172733Z", "generation_id": "gen-1792957501-ndEs1takjha5ElF4RcLS", "proof_sha256": "5310f82ba42ce8b3b20f01803bc646d10eb99fab362c09de17ff478adcc2b8e8" }, { "model": "meta-llama/llama-4-maverick", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792957518-rWOavZ3bij8QjqYDqdM0", "proof_sha256": "d5454f1168b91c2ec7314a163d043e70766079f9664ec88f770b9231ce724dfc" }, { "model": "meta-llama/llama-4-maverick", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792957624-VbKjW4HVeLKfkGJCpnDg", "proof_sha256": "62b0155dd7beee2b218c33260734b29b89ca75138bd7c1c93ae6aae7bf2066d0" }, { "model": "meta-llama/llama-4-maverick", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792957700-LKLGmmqBtlSKnWX2JUva", "proof_sha256": "c932fd4049889173f2c1fb3631341ee23cdad36d5a781b49b6156e87f352dab0" }, { "model": "meta-llama/llama-4-maverick", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792957762-l3xCDQRUrVIrFXdvbtqF", "proof_sha256": "b3d1d47e7686dec454bf4a91f70238f5499b5b2d52ff8f576b13cb3567b91813" }, { "model": "meta-llama/llama-4-maverick", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": null, "correct": false, "errored": true, "refused": false, "excluded_not_in_manifest": false, "finish_reason": null, "host": "DigitalOcean", "run_id": "20261025T172733Z", "generation_id": "gen-1792959359-08Neq88BvGlhcgEn00aj", "proof_sha256": "16c145803366fc6e9c5b941a5f7f3eb59799d1d8b3906a91ab5222d02c1da55c" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949253-2tHWOi7Flgv9vR3NJ4Hz", "proof_sha256": "dc49be58f909a4501ff75606d84a1c97f3511ba12e60a746db23445f9f3460b2" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949287-ExtVwJ54hePOdTYJVIck", "proof_sha256": "a42d8b9f4d5b9e45531c5ea39378788d4174c1f6905b1a9bb9018b339df6ebca" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949333-gMOFLkUfTNvVAoHDMR9k", "proof_sha256": "f6eea7381d76f558a9eb9f0d81ee392192e171f9ace492fd6f38ceef1161c0d6" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949349-p4ryfbxChoHdVvmXsVhq", "proof_sha256": "3aaae1c18e36a2015ed80a0d71b0d93fd70f14b75726c556ac589221de272b11" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949409-ApSMcXDG97GhRl79aCsh", "proof_sha256": "cbe54b9506bc52d1d245ea21f7a70d53a7ac34f1127f18dbe1e2830b8ce08d92" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949502-cw0KnABWjT6GldjJeObP", "proof_sha256": "3797376fba743e6285e083c4ad7aee954092e77357d0fa344b4b93f61b6c4284" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949531-GFQz4ldkmmVHaarhWf6l", "proof_sha256": "e4bfac6e686e12daa04f3c88d66299a6aeea7ead31be13df7002867191c8d7a2" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949559-wMaZ2wXNUiZMHK8oM9cV", "proof_sha256": "5972075d39f2faa46f8735dc809409c57ed7157592b0ba0eb0fa0ecaf8eb8989" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949599-1W63tKdWadOon4MdkmBt", "proof_sha256": "9c37ba2b9766c7101c1b518cb5aedea4919ce218a43d67acf2cbe4b00de399db" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949626-0gGv6nIpXXoUy7zep1Xs", "proof_sha256": "bd2fd171259674b2f5bcb1ac743f2a8cc5c017187b77cc4f72255cdce4d2549e" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949703-qX2Cd7mP2VQjnsSqv0wn", "proof_sha256": "8e9ed7218543ed8826d907e1e8b65b966964c3c34b7fd4d127e812aa75861b6d" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949716-FWNsgR8ldhyQaBfh5yht", "proof_sha256": "22244ddd2a90a3304737166079e7b2041d2f3fa495b12f47251b90878c259446" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949842-bJQAA1vDOUJSSF29qHzG", "proof_sha256": "e6f92f499e29a419e5ca99e95efb55cd4a58aa5ccc9b1ea3efe3767f138e907e" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949856-fEzPhM2OmlyFsC0meEcj", "proof_sha256": "f2c91a7ded5eeafa98cddb23b15e1f722724d7e13e2356c548991b0e0e990717" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949886-tgeFSwz8k2MbhjDX9Lcx", "proof_sha256": "21302e25919e7371ea5daf03441a6362d15ee8199dfdc48387b11e380a8e3f0a" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949952-bDrfFRjcqhDusHuzbSWJ", "proof_sha256": "1f8ea51468ddabca9ce01eb333758e80bb49ac99da41e1e027c0b04b6bae37f2" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792949996-kfvyYaAEwKLxJPk9bqjj", "proof_sha256": "5941d357d0069c7ad5c51153c9d04f7f829d6ad8fed4adbab931cbbacf7da378" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950024-M8S39aeyWlLOoblNiShn", "proof_sha256": "916123f46f8c23a34e9b76dc023142d46eb8e28eefcf76781278ed9042c52256" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950046-BRHjI9K0s9Ve4vVgNb3i", "proof_sha256": "d5633f9eeb13f8628b4a0749a381e7cedba593ffd4fd46dc1124f99af0f336fd" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950076-qCKg9vSgmOvmQiJT4Yhs", "proof_sha256": "3444659ab5dc1621167729acbbe4929cc71a5094cbfc50c49d30c1dadd9534b3" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950101-m8Fbi8bwvOMCSXCj4LMp", "proof_sha256": "8026e5bcfa041c179c1cc0c864290783d2ee197615e5e33db54eb72e3d7f8fcb" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950145-MUZw67vxTi8O1jlqVqI9", "proof_sha256": "6a2c00e80030c1f889dc65c9c73c9b84826ee2b7d4abe2ecadcb25df750c3f01" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950192-n7h0vl6rJ8BcLnguCbUe", "proof_sha256": "a9c8b261c210dcbf4456f8fffa749b0b61d0f966ed0d498e0f3bf1626458acf4" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950329-gxYaRCSa0Pl6Yum6mmJk", "proof_sha256": "ad38b0d1593ec2483118b29711969516cf613249fca9036d931262e4b4504c13" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950377-BC5hLcBr9HRIiSjatcWG", "proof_sha256": "17f4dd7d19ab568cbc966b0ced1736d6a1f29596d817eecb1af823a90bace79a" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950458-MmAvhHqEIdYjlcbCfMzR", "proof_sha256": "6affd7545d5467557b90e24d7e30787de348eb1b4a948980db7736cec6f33e7a" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950498-XH3l3myOgCHAmJ1VP9KI", "proof_sha256": "81bd5c079541548f8f1805f2d33e2aa8a55b096a941b12fb0a06d2b690892653" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950536-4hiDICCGfL2q0hdaz7Qp", "proof_sha256": "f31ceec4a90efbf5ab22fc6602ca88f85690ad02f989f11ef98a02f8405f3863" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950561-kdolORGjNK9SjMvwuuzN", "proof_sha256": "4fd150b4062d2aefa7d97306976ef6ada01dfadce7758921da2dba34c5555eab" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950664-cHA2oy6sXWwtvDVEzts1", "proof_sha256": "15a2a25659f94a021879d517e728409fe5a1c05897da0134653704a1e78ffd49" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "\\approx16.890\\ \\", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950687-OebF1DLTRNLDhH3hYUnr", "proof_sha256": "e020528598d7cce614c76dd27966ca9ada1e9145e11afaee0f4acfb0b7e3faca" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950790-x3YKCNHYz15qWfdtWrh9", "proof_sha256": "077d4cf306209c77022c57a15b0cbf2b5124c1d53d3111c4f0e986ee60a1d46d" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792950844-0yZORPcFHGdgykNkhZqs", "proof_sha256": "0ec7f69f93814bcdc8352c1a16a4110d6b9abb06cf52f8d961c29df030bc4106" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951866-8LbpkOlcmeWXNv1dryBY", "proof_sha256": "1651df8f839e971867aa8eb8aa3432eda92f9750dcc8f05cdfb608fec89bdb66" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951945-xGpzav1E7b9kGMSMEoe8", "proof_sha256": "fcf0e75d09b5a723e543a13de8bb8c7b8ba25a736f7a23145dd01fe0ec6d30c4" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951976-nF2cl4rtTkLIou2rKTkd", "proof_sha256": "9ef47c41b474f5a1a1267523621345659ac632f456fec3026b62df0a24d8fb3f" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792951996-Wi2iQhAgeotiGlnyAAQk", "proof_sha256": "506faef65cff4268ee6ce42cb96767c9f9ba123f3bfe4c49ac112f587289290d" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952016-ywL9e2vhXYsIHYX2rtDw", "proof_sha256": "0b795aef6bb065b2c7ed18e054121751ed55db70ba6bf6515dc61eb70af61711" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952033-dP7qqHvBjK0qGsVuUuQF", "proof_sha256": "7e44edba78edbb18f72b03830e4fa1becacdd553c55fc18b9ee81df7bf2e78ab" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952051-BdncILl0XzEi5NqRKWSr", "proof_sha256": "6a42d7d23982725105c61743cafe0dbba2e48aa71c0a7690c371653483c881dd" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952068-M0HQzsJUnoCk0h81vKMn", "proof_sha256": "a1eab751e63e61d2b1d65485422528770f0d4eb018ebad91239920fc42283e70" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952082-Hr29yrey1ELNsH0ABwe1", "proof_sha256": "42dcc718c343364f68480d2bbfef8a50add9c9f01e5681f45cc95d0200f8289b" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952094-iDhMcxziGrPEKM3xnLQK", "proof_sha256": "46d87c1ff3707a7e6a54c0cc44b05be539f7b6bf9d2e2af76b93f4f25815d5d9" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952122-RIokBN25CeVndcSG4LwU", "proof_sha256": "01c99e39572d1d8786e90caf24cb62767a554731d07319a2a90ba710abd6ae45" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952226-6NckXUNA1i4pPaSTjLnP", "proof_sha256": "b340ecf00fb98669de56c699b1708f1ad9cfc3d4d889bae93ddca1db92c35c39" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952248-x47Y0dO7ZLYSw5adXnDU", "proof_sha256": "1275b6bc7aa9baa7a450a556a25beaf67bccf7dcdf65e1a04af94c3e24ef7281" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952276-PLrUBi2J0ubvD7HjCinV", "proof_sha256": "696731539b145934f23bdbc8f40d85f0630af0e174786572ef9acbb9ef89160f" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952297-QCVfYO7o594cvtfhl842", "proof_sha256": "379e3650db915346229c5ebc8aac1933649695f477596833d79e97f40ca1469e" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952345-ica3NErbo5zhgMPUVuUD", "proof_sha256": "ee5f19532ae8c7fb5df16c4f180e3800830f53fdb92dcca20d2afecc29863fc3" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952371-G8JnhO76KPIMYcjNlLhq", "proof_sha256": "ecb3a8b57469f139774afc9db143c943d7d710d76f9d6964e00eae6313ced47c" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952389-MwqQezcmTpivrc0qH5cN", "proof_sha256": "762b510b9406d2faaf24b9b14ffcd8d81d830fac5a7efe0461e9b429a04bc40d" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952496-NsalsvbdyOS28zLkd7pc", "proof_sha256": "191bfaceec7d2971a6f8a00d18b490e317b3710fe0b6867fcfc67a4b2fe34688" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952527-n0avxgf51zDeSbbsMzKe", "proof_sha256": "1961d2acf64981e80af86560cfd04ab11eb38c7a1f6ccba5f4a8ad08ace298f4" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952556-9F56Q3VDlXI6jG7IGofz", "proof_sha256": "e80f34efd47a3f4424abd53df46df3339a8dc6ed06c5a6260a8d224dd17777a5" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952584-2U6AZCqq7Osgt1WMlBCT", "proof_sha256": "776e6ae219c8787be034b8d90444286eb151eeff82c29abc8f2f07cde85464dd" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952635-Y98sIZdohZVL25hh8ChM", "proof_sha256": "91cacb4f2ee2d68474f76c405d9369779abcb387860c0d5e316a872c38a881f5" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952663-E8KVs5icmSzmYmSfcT9F", "proof_sha256": "aff9506e38315d66a417c929215e2b82c8aab7bc5b38cd16555b747d8ea63aa3" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952683-MWTbGqDDJpuXW2DHaLUw", "proof_sha256": "6cc02e9aaa1c1da742d36ea06caceb888899676d4a06d674a94b1c72f4f09dc8" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952703-DpOzIanFsaAMIvcknOMs", "proof_sha256": "08266c32a8a559f820063d5fa672ddae2ea1614a0b5807ff7c3debede695ae03" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952725-cknCcRn27nqy1dZ9thUB", "proof_sha256": "517b609342d9080018437db0073f24223094d4e2aebfc3f3e856a6098edc9ee1" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952741-ENQBhyID6wkWd8ijZyP3", "proof_sha256": "60c3169d0d54982e4ef35224a76b07b2a95b4ab5721e1986d6563295cea94321" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952763-iIbt8ABQRcEncyRLEdYG", "proof_sha256": "ec2f16082cfaf510779d8272787d70c5ee929d97ad789789a0f30cc496a3a9ba" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952784-BFKvAzKAjDVzFP5UZk75", "proof_sha256": "a17570808da6fd3bca5e83382f1ca23058f2885c7dcfd8f65965b0e25e1b43c1" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952816-sSib0IHKq05chGKqearT", "proof_sha256": "36d672cf26321a799eaa74621ab3692e8e0a991fd9b906caa132d84f22c66d09" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952852-dCUOzrgYdsee7YEVQkhp", "proof_sha256": "161df1b08bd638ea55189e04c3cf444b4ce530d59252e87007ff36ed3b575ef6" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952862-gEyHDPi9WeSVqPebRFaI", "proof_sha256": "e64365dfecd0ed20cb360f55e6b773f4e20ba6af2762504b9ca195e507ae6fab" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952881-yC6JUovCJXIGj35euog4", "proof_sha256": "494622cd11715b6c313563f70e9d20c63046678aa75f20c78963d8e1b8037af4" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952892-U1vxdPP8T4YNoqprbaOh", "proof_sha256": "66300afb36ab7d5b0d51916c29a9caf93a111a00af2f59ae8613fb707405dafd" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952915-x6zg69rldw4I0mdu6q3E", "proof_sha256": "672608329cee956b613247c02045b10c0ebf073a154e7f21b95bb67df7d3b092" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952927-8WiYhfCINoaxfRWQZxM7", "proof_sha256": "e4e3849f58c74a002e0d8d056742bcb5ae2b615c1accbf3b828be84fece375f1" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952939-CnXrYAIRww6klU6He4E1", "proof_sha256": "ce2a9e7e0168db3f7fd24cecd657cd2bab93f9d4c4e616e84b70344ab94907fe" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952949-O5NF9Xtmflo0jx7v8PVx", "proof_sha256": "ba289ba42e459faf99e8f59fa6ad22cbec3d88899d81a737bd0efd553c17ea98" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952988-IHzrSXOBZW71GfD1U2QM", "proof_sha256": "c16d898d72b7488f54b5115050ac7774b47396768d32382b3bbeec6ed9d96a6d" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792952999-vzZcoDgE49GM71gExn1b", "proof_sha256": "7eab7c5236f5938133161013becb357f27c6f4d04785a9c0d4e3a01c28c09ffd" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792953015-D5sKydYYRxhNwQ1kV2he", "proof_sha256": "f527cbbd01ab8306c26ef74f3db9dbca44c18c3abe350a615950315762fdd8d6" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792953028-1wHDqDY9AsHlETVBwUft", "proof_sha256": "22f268a8c9a8e720144d5d65fc80b3341880c272cff7d9b2add284eacfc43840" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792953039-OyQyGthY2OFyWDnmg7RN", "proof_sha256": "60a78cf40cf996e9f5a9b63097641ab36519de4eea4279b84647bf9767fe3994" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792953065-gbZtRb6FbYjNysbXvV96", "proof_sha256": "770c259118e460a705b1cfcb3dc413263e7f65f4d201d89a6a21f25b97e4c794" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792953077-jILoHaS3zE3HeqWAcTP2", "proof_sha256": "5335e12f4624b70707b87e50ff38aa4d2149f32aef453ed727babdde9feddf07" }, { "model": "moonshotai/kimi-k3.1-pro", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Moonshot AI", "run_id": "20261025T172733Z", "generation_id": "gen-1792953089-c3VKnSNYreTd13mwEjn6", "proof_sha256": "3308b27db2cd16fdebfd6099f55100fd0b4d0e5df15558d21787a86a7a28c789" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792949253-ffr8SjKydhW45ThkRa9J", "proof_sha256": "92c80993a231f790acc83379374eb00bdac5758eabbac4f24a4206289f1d93be" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792949291-TbXPCgrlIRLu1ctU6ulb", "proof_sha256": "4e2e9fff2589c0ee16b7cbd864205b84ed758c12613ab37b1582d560e45885b1" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792949350-GnXHQkKr2uvqFCnIRUB0", "proof_sha256": "ab6f257a0ef289d1db15d2934a5adf9311449119ac07c16fae83a78d05272ab2" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792949393-psITONAKQ1bAU55yj2SQ", "proof_sha256": "128e16d4b13a00c3b1a263e149bc155973467d8bdc64caf3189c21431f8e9261" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792949469-vZ77oBIIIQJ9wdOi1YkN", "proof_sha256": "c74bbfd7fadb949d1efb3e0857eed2ce7fc87d8b6a8931aec609edbdaa2d9f47" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792949541-ddErVgMCQgvdVAsS0cZR", "proof_sha256": "772334fbaaf96c0b4e09bc40edfbb0b5af8b9cdc106dec1dfdf3d69416747082" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792949593-82gvRHCc56G50bYxZc86", "proof_sha256": "afdbaf8f1be21ca1697423d40c8105815ecf4cc41bba675ef6dd92e6f18f95e2" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792949623-RQnTOh4RKOCz7UU7cCDR", "proof_sha256": "3901b8791c358e1f16643c405bfefb0893196eae3e840aba8a0deb221fd3be38" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792949658-a5UdxlbTY9JAwJvs6XWK", "proof_sha256": "20f7d7eb67bd2aa243c1c43748a6c9f95737ef7b67ac7a42acaa9808f6404d8b" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792949692-ZGBrTGakGbmZjxi0NPrD", "proof_sha256": "6dc69b7496d1afd8becd8d12f6fa32df8eeabbf60b95824422882c3355dcfdfb" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792949776-ljCPEIzWL1cF2UGQ7Xj2", "proof_sha256": "a103da4e3b82902e2b2f7d0193125cb5c34daa5e97b6a9728932070b804cf251" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792949811-jqJIx16UA6nf9EL3iSGF", "proof_sha256": "4a07524c017d8b01361f07f07deecafaf172f814fa5be903a8b833f69f12ca97" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792949894-LsPTWjbFhixeB28NELuW", "proof_sha256": "151233e6cd8380fc311c4844315b017781c5580f78d6030e54546238f9d61cd2" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792949920-RZO93RdNVBmsQO2m8AlM", "proof_sha256": "e784a8cac8ea2a6292be18e21ae4096214dfc2244ee82e2cf6de03d8bdf09952" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792949991-C2oQDmVv1zGM56gYDgIe", "proof_sha256": "4dc5ecff88490eb25751fab5487f662faeed0665a9ff44ead07b8eac229f25a4" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792950072-P8U2HnCALYmMC2ueq84C", "proof_sha256": "bb8e9ff2abac2a626af7efc6ae82c0bc6f77ea72aeec380daefd2c2c23d3970c" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792950114-6LQ8wu342DK3VK73fxLy", "proof_sha256": "4866113eb772a9e81782cdb38b9fc85ae386dea525b24c55445fd80335ba0474" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792950181-rc7mHAB1l92zWmPBSgtb", "proof_sha256": "3c83c9bb317a7708acd45353a25ba85492c1957f515d7eedcc49872c4d25dc75" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792950246-8Ide2g6pAYI0MLVkGfyx", "proof_sha256": "8c4539ccfb9060fff009d84381313a91422f4071a772ae0584f9e7dd91a89099" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792950304-cOb7q9aU4li6MszsQNwS", "proof_sha256": "909d271d1e40cbfdcde13fad826196a99094e7f799d4be8c2dfca3a30af0b03e" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792950357-qHntjvMSKh0eu5dSU95k", "proof_sha256": "02861d284d0d28eb5d67bf09ac454e5d2f6d542741eb4c8c15b24d3e157d2502" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792950388-BX0vAwX5zGVXhCpk6k5S", "proof_sha256": "444eb81f7288ffa4f6bc511cc9af3ff0ea01cd50507911ba31e2e9c39ea0fab9" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792950433-jsH0Dh3S7zm9uRefoYGx", "proof_sha256": "c04ff4cc9ada42842f3c47f93d878c8a414ed267d593218cae0695e5fee74587" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792950520-Y2DRHEm2mUf2CMjOZ9jT", "proof_sha256": "1a7eb45bf7ad586fdb4ca4d17fa6ff9ec17905662958270fc4e68251e5a2fc36" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792950557-oJqIfeyA3AzNcjp7vKu6", "proof_sha256": "604742bbf275a253a939bb38418ab37295948e371bf51b0a267b867e1b34e44d" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792950636-VH7EUHJjQWODTOSwScQq", "proof_sha256": "4ccfe7a01ce0e771906dcd7818f17024a0199cc2027fa1001dca95983016de56" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792950697-1oTpjHbKfM1aU1gISXHy", "proof_sha256": "58eb22c741f914a0c41c02708f3224e37f348b6d44dae3402684bcd47605df17" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792950746-FmOfv4ydqgwSx64LnZmZ", "proof_sha256": "25f1b9fb89ff614dad16d541ae42794e8f0efe3a36d2b8669878f79ac455e870" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792950825-4rPdx9ayHE77apPHy4rI", "proof_sha256": "163581a54fa7dc7ea44ae1da18354619c1fef0d3b5162a6d52a582c8672a6c82" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792952164-Iyx71IZtzOkAH0Pp3FpA", "proof_sha256": "9c01d1f1a467249673b460856b62f491ee660d3f0c3b9e186eb06fdb969528b9" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792952207-Lzl7Mdn8dcY5fZeajIZO", "proof_sha256": "1e17f9b6e71b9beb1f56dc1729a63104be3ce146b708a373a6fdc92f66711691" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792952281-0b3slEISQyhWKCFVyh5I", "proof_sha256": "a4a0186d01a9fcb43f1df4b614e5d090cb9c7d747888d8e1af8b1a4e97883070" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792952324-9s6QhUn1SsFuauYBmo8t", "proof_sha256": "d0325485d16d95117d0213afc2ac6f1fec6e2b710ef36b209c6ab57d0972b887" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792952390-q5nwqBoid6cm8M2tpCcZ", "proof_sha256": "0a0a6b02107d8f581ff516f5fa921ab228c5d4aa544cfb5ee64e1155e29c2a65" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792952422-yiNtQHQxgXH4fBvSInQN", "proof_sha256": "15047984f8ba524f2e7c5ae8f9dad3fc628f332a70de6bca2ce62a4e0e2cea51" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792952450-ONHPZQ5fydgMfYUmQ1zy", "proof_sha256": "048dd3d5332f61c23b73a3bb8b69d294e962d30b3088e1de9847c33d34a84985" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792952482-pLlU8vXjm4QLn2TNjjjd", "proof_sha256": "657d94ee8dc09d3b4647566bbdc9f0529e605ecc125a51599fda9ccd0ae31e25" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792952531-k7JG7lKzGgEOAIQdiExf", "proof_sha256": "9002cc4c0df9c9c66b6f5b596577adb1ddbc31e0fbf70a81e8feed11027166a7" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792952577-TrzG59Ottyjee5DIawzX", "proof_sha256": "0922a6a6635a9cd4dc0b0ec153ff4ba93a8b773cda382401f0ebdfe87c1dba7a" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792952608-f6O1oxxES9bvIXsXlFs2", "proof_sha256": "7262ec810157ca06663483b6614cff567fb4d3bffc55a1342f0b678178a4d4d5" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792952642-sUZ7JGF2xX19XkRhPcYz", "proof_sha256": "807067b0bd536a288dcbf5a2dd4c700d648cb04478c528f9f01cb9f8a1e1cb02" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792952683-HDHYd8i4WM5Z9mjI5bJ7", "proof_sha256": "7a615980c9279c892c0fe3d7d9eb669fad331d587c883ad0dde35bf4f950d052" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792952712-qNKbinpdsaG3UXgNQZ8u", "proof_sha256": "ae069b1ac378ea3edc19905d6dcc03c9bc47befff2d23f29b61d58b335e2eccf" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792952763-WtvnOb7rplfSMR4qyJFp", "proof_sha256": "70f0e07a198ac2b754aa0655bc60f582d33a257bbea604d7582c250d15e52e1d" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792952852-s4J0jnoKxmqR7CgGnVpR", "proof_sha256": "3de210bdb4fa75ea7d0cf2add5c1e68cf0ce9d48a11e36cfe96dbf7e8dec68b2" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792952885-pFKs0QEA7Cjvaoo32NJP", "proof_sha256": "12fc68fd84d5e2e62718b09348cedc1100b18caf3ab1bc2534a718e09a4f9591" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792952920-3q184m6RpFffnfQOFjAN", "proof_sha256": "1229412b8c900eee9ed4a64cade31bd500f276ddf2ba289d521c7d226a58d048" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "length", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792952946-cQNVgqWz0WTzWPUke9Zv", "proof_sha256": "647cd56a30b6bc8d10849746eee60c3f3003a05fc6bdbe0b8c608718dd4e5778" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792953022-KrGAKie0dlarvwBpmTZp", "proof_sha256": "6bc31dddbbd7df00af9183df603d7326e6b0acd438f42d783bdc916da8ca3e33" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792953063-n4dXFOKDxkTzORomUWZ4", "proof_sha256": "77890ec660665400a52a8b184af867690e568bb43bc55e4f8be456755dca3fc3" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792953091-bUpE5z7yUBFApqFxkYGU", "proof_sha256": "c4b5fc6e12c74511ef3959c8842a1ccf17d627e7ab0a6241cfb792562fd6eb7d" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792953130-buK4kb1SGE4lRWvgu1Tc", "proof_sha256": "3e2ba9d0a0bc750b58cdbc34c0586660ba27d4a8ad69b1405fb95c311e63ce31" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792953164-rx1FhxWDSrA9u6XfVX3R", "proof_sha256": "799e99b6395927ae51da301c53428b0115a7053623ff1b47d8744c5e28f762e9" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792953214-Hzeh9JGks3CaWL0SeArn", "proof_sha256": "5902bedbe9939bb4131a308243ce93e7facc77dfab117910e13146766085aa16" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792953269-lFe0GFkuHweBU94Sjdxj", "proof_sha256": "73eda7852f0ad7b7a329db6e05e8921e12c3168fe0320b73b360405928612d3b" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792953305-Y4x5n3g4lsbOCt5bA8or", "proof_sha256": "0667b0e1240947d56991f86a0344fcb08c2932b33503e74f44695972166852f6" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792953323-lupatGz39gVrxK5XTC5T", "proof_sha256": "3e0833f99232ae65d73896b5fcd7208029346798205b8d2fd3a00516284ae591" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792953343-Pvf6PTWgbhQc0oROV9wA", "proof_sha256": "39d3fe5861f19403f63be3b98846b80f5ed425e6674f8cf77bedcc576d4e5337" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792953370-vk0jMz5e0c2iVeEPwi1m", "proof_sha256": "6a61f667e87571ab6018e3ad53a82a67d9b516343be4b75f6d2968103288997a" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792953392-wqMsRQK5rkayzxuQgFpz", "proof_sha256": "10958be0e88243e5aceea7faed60f4653758037a98fb016a1fbe38e05a399c3e" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792953423-SibjSMsRX3DJpI8GC3FI", "proof_sha256": "fd49bfd3066d97986b0d8049fb7cd593d0e5bb7991900f992956fe2048ed33d9" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792953457-mEd2XHYqEcycNygzyM7Y", "proof_sha256": "76f27460b97734051d78b604e8c0badb9ba55f679658626e90790d1c5533dc04" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792953483-pMhLz65xU29N11Wrm7dG", "proof_sha256": "74885c2510d6e6ec0f3074a1c9f35d852bb82eba06e524d827a994ff6d055ee7" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792953520-rhdlGcLBcUjp77J7do7f", "proof_sha256": "9c48e2a7ff66c7a5e49ff04474b6991f8935d5bf3e4fe308206485d32c53bc01" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792953536-FfXhJlCxk4TfWcNZkJo5", "proof_sha256": "011fe0ce1fc077283c92edb6b6a79201aa9055628f25e33f968d612bce110bb1" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792953550-kaIPJQTXmNnxYs3a5aYx", "proof_sha256": "78d2aecd65e5e20c65f1df384d39ce64e59da0b5945c955e78dff6094c998cdd" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792953573-EchDsXKYblZjEq8lOxqE", "proof_sha256": "de38f68954bd2fc7b19b6b104a62354cf1f812c206cc7a59caa8e5370bddf70b" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792953594-JKuVOZmOesrIsGNYx955", "proof_sha256": "3e23ff31d666b7724faba77b15043a183f3f8162d69425b26c697ab696fb7543" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792953638-CEyMSkZPDpIlSBBssAvu", "proof_sha256": "cd6e284d03ca732b91057b446628637ed6df367a887fb3e1bed14003d8a6508f" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792953659-DPIcqnqKZm2tpMPJKw0t", "proof_sha256": "46529f5c8139df05f6760a8309db25e808b7321141087b04d7136037c138f773" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792953679-H4yaUJ2H6C6bvzXzi0Kf", "proof_sha256": "27c43f2729df1afa1d471e656102a80e91598b98fedc2604d716fc3cf02b2592" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792953704-FrvehO2lW128GU7LV6Ba", "proof_sha256": "de0289b368420bedade9b9fb81460872c929a2592fc56bd7ada7bb8323ccaf6c" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792953769-5CMxXbfD4aqw6jfDGl0L", "proof_sha256": "29db7850949942643da8e35649bb541b2418e0897d0c1fe551cdbc5069d23599" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792953786-ELGuV4hp2c224BCefRit", "proof_sha256": "f299ff668f64b7c298d6e1d9e7e17485f243ee1c26969bb57729d040684d06f8" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792953817-s3sQq9u6NyvxR51XBwZB", "proof_sha256": "a3aac6c2193c12a2047af0faf8762b08322e37e38e140866814852578f8b1f3a" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792953843-H2H51Ab6e9cRsOng9lW7", "proof_sha256": "baebb659bb27fd4efb8d012e984774b332bb32f50f5e41b64984fc5be9e22e85" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792953872-s78iNC3gSKP6Y4H1Ah4w", "proof_sha256": "d5eee7d4f623ac87509fe594b6970d8777f326d773aef2fe67daaa91b23b16b0" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792953902-AeV6F6Sivk6fdIWCpk2O", "proof_sha256": "c3f43dfc43dd9d910b4d9cdab20fc645bb0e4872c651b4082d3ea67142fc3395" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792953929-uhWjt7Q5DUu1MiOXrQJA", "proof_sha256": "6d08f232e52df2623925fc93cf5dcc87a694a55094d5cc0d5508078f99e81ab6" }, { "model": "qwen/qwen3.8-max-0902", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Alibaba", "run_id": "20261025T172733Z", "generation_id": "gen-1792953950-M2aRu6AVYVqtyJNUfuRB", "proof_sha256": "45c9cc9e8b47f5a1e130a1fa20563715f8fdc91f36eeec4a83620e2354efb244" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-8d28e5d004", "dataset": "supergpqa", "dataset_ref": "8d28e5d0044247c59ebeb7909500d1fa", "category": "Science", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948012-flgdchAEj71tUL83pO9X", "proof_sha256": "545a178180230444ccae3b36b9fd786d1438d5b98a1915bb2b7e49e1f04d46dd" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-7c99298107", "dataset": "supergpqa", "dataset_ref": "7c99298107974d3981c22cfae13dad07", "category": "Science", "expected": "c", "extracted": "e", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948032-F02KOoZHzxyaYrpKl5Zj", "proof_sha256": "ecb5493c7868998c5a5ce65ea9ddee8efe1554612d41100aa2d43c63ad5120ba" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-ec33474441", "dataset": "supergpqa", "dataset_ref": "ec33474441a246628b52f2cd92e42848", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948054-f1dnBxJ9pMuYquouEgou", "proof_sha256": "a147fe25fd272a6333ee6315ef5f9eda8b00b79f51bf69f8f72101a7574ece58" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-47cd3af3ba", "dataset": "supergpqa", "dataset_ref": "47cd3af3babf4639977e79b14e058551", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948064-fnQUXbS5iCLzJcrTJCNf", "proof_sha256": "17548d0cf57b889200d4ede936fffd8fb145ef2bddbb5cb7764a837447861ec6" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-3f2aceb643", "dataset": "supergpqa", "dataset_ref": "3f2aceb643cd49dcbbfeb538d52f3c38", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948082-SjMmGEy9z2ZD5ZooEGYS", "proof_sha256": "09664d2c9deaabac762948c1c4f483d3ddbcdd247f3165346b4df60d7704e020" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-cebe690510", "dataset": "supergpqa", "dataset_ref": "cebe6905102348618c9173144213050b", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948097-T5kRKwcxK9kgEzAaKq46", "proof_sha256": "2888664c5105f6fc224a9158f12280feed33886c907f3d661cbaf928e1a9639e" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-d74b099f57", "dataset": "supergpqa", "dataset_ref": "d74b099f57dd4cc0bae79c165dbe4540", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948108-5W54g9iO5csCsBqYxD2E", "proof_sha256": "c31a5c50bc579862769de18259d0971a894e5f11a901ec715f1e8bd58b1904bb" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-b24efcad05", "dataset": "supergpqa", "dataset_ref": "b24efcad05694b829c95f335cb72bb5b", "category": "Science", "expected": "i", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948117-qyKfkbZcDHDwmKgcCau6", "proof_sha256": "b7d689cc28d2a169e4a22778e0b0a2f7bbd95dfa3b26bf7dd4e598fbd2719600" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-93a94af4be", "dataset": "supergpqa", "dataset_ref": "93a94af4bedb477ebc39090181a04e26", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948132-ql2BLpHdSytRDI9Wy8Sq", "proof_sha256": "6c8543957b16e77679c25117ea49176981cf839e52eb54507679c98e3c1396b5" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-14fea390ea", "dataset": "supergpqa", "dataset_ref": "14fea390ea3746d3a5610fdd4d0d4175", "category": "Science", "expected": "b", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948150-YnOL7etIl655NgXYRb3R", "proof_sha256": "02f81c804ca499ee5468353defc6e0175de6627c65f7c24e4d0c0bc243ac7d4d" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-f42d8cc728", "dataset": "supergpqa", "dataset_ref": "f42d8cc728dc48c0a7bb3078ab17aa17", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948162-uUgcuMGCCBKaB0G88OK8", "proof_sha256": "ed5dc1f0bdaa1c036a20835e86af41b3db2fa7f6175b38e11e0ce19c7a2d422e" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-878c1738c1", "dataset": "supergpqa", "dataset_ref": "878c1738c16f44f8ada5d025021c7003", "category": "Science", "expected": "a", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948182-FcHmb9v0WO4ulqW5TELr", "proof_sha256": "1d8b045f958cdcdbf45c834017a66e7f326d1aad775b9afa2b89653bea1f59d1" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-78081e02a6", "dataset": "supergpqa", "dataset_ref": "78081e02a6b4406fba6726ff68fd7efe", "category": "Science", "expected": "f", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948203-2zv4AO9MlefWWIv4JdtM", "proof_sha256": "8b9c4e9d18fc54453b4d37dae9a3eaa55437f98871bb5c9104ada5c290bee35f" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-2f5f6db254", "dataset": "supergpqa", "dataset_ref": "2f5f6db254e04720a180bf58f1dfd7d3", "category": "Science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948221-rEtbMlHF91ZSOWlnkuKX", "proof_sha256": "0d3a2fd2ed51e0977ea6d700200073d6f3d2b650bcd66a28b67af0060b1c1d31" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-e716eb0e8b", "dataset": "supergpqa", "dataset_ref": "e716eb0e8b00476da24995362aae0179", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948230-Vkg3er0aLv6mt3ESirNt", "proof_sha256": "e6139dfd779081173434a415bbc682f2925fda616d68450d394641ff5b1ce2dc" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-d14d338a85", "dataset": "supergpqa", "dataset_ref": "d14d338a85264ca7ac2ad2fb159e78cd", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948242-w9h1ZaNqmY9PNuWmpU3Q", "proof_sha256": "97736b1333bfccdf1ac4de236d5df4e8eb44546942445369817e3c3061abf0ec" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-2d7381513d", "dataset": "supergpqa", "dataset_ref": "2d7381513d204f5786ba6d6a1dad6d6b", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948251-kFBhYZ0jav9jAaT6ysM2", "proof_sha256": "3257e4668cbbf45ba50f35586de7577383115fc48cc53805de51ac698acd814f" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-0fb66d16c9", "dataset": "supergpqa", "dataset_ref": "0fb66d16c92d4acaba64b3b0ebe08a94", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948271-X530sCtetGF6gK6jiThr", "proof_sha256": "a3684c868873c8e4572d96c864d46cd576744909704d915e3d5ea77709510c24" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-6c535032a6", "dataset": "supergpqa", "dataset_ref": "6c535032a6eb44e98af1646a3cabc2bd", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948281-vbznBPYdhrJd5iTgsoAV", "proof_sha256": "cc49859807cbbff2dbab8d59c8b1dd258ff2fa92a3ec86bdae0425ace0f3b790" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-91f13f3c51", "dataset": "supergpqa", "dataset_ref": "91f13f3c518e40a691069f1283ef21dc", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948300-PRwML3LnJT76bwtbtUbG", "proof_sha256": "073acbbb3209fc8e59d2401663126b7f1a88e8a39e08f3ea86ab300fda71c310" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-81712a85d0", "dataset": "supergpqa", "dataset_ref": "81712a85d012442bb89f380e97d1591d", "category": "Science", "expected": "h", "extracted": "f", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948318-ybjOF2Kxxw5N5x2GRc14", "proof_sha256": "6723acbe4dc1f071bace049b1094bcd510c006a51ea43e9481d805a2f0a47ad0" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-bf4ba55f1c", "dataset": "supergpqa", "dataset_ref": "bf4ba55f1c4d4341922999e7f97105cf", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948336-5kUByC2p9kd3pnQAsULO", "proof_sha256": "bb1daf67a7d1a7a174587f5bcc755df6eaf5ecddcf33c8c87814d16316b91067" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-cfb3739553", "dataset": "supergpqa", "dataset_ref": "cfb37395539840d3814e783a72da4f78", "category": "Engineering", "expected": "j", "extracted": "i", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948346-7Cr7ld2VqfkFvnOYNHu8", "proof_sha256": "4088ce594f8e632f9afb691ee2e7236b331601ae6e8cbea0d7be09678eef5e01" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-135e7ee0fe", "dataset": "supergpqa", "dataset_ref": "135e7ee0fe6a497d9d78b4c7d9d995ae", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948361-h0lRyHU4xNA1ULsFb5da", "proof_sha256": "8fd7a5892d46c2aec94ab8f9775c5d653d52753330e525050bf5ec09bab647ee" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-fa2a7182ba", "dataset": "supergpqa", "dataset_ref": "fa2a7182badb49dfb3a8fccc23b5e99e", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948371-izPldNVnX64k3O6U7Fey", "proof_sha256": "83954047c997e12b00db48d89884458b234df507c7eedb5bcb1139bdcc96136f" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-6a9eb603e8", "dataset": "supergpqa", "dataset_ref": "6a9eb603e8fb43efa6fc391c18f0adaf", "category": "Engineering", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948384-hdrNdlSeOPnghbywY6tM", "proof_sha256": "c0266789bece2064075ca1b1dbf22f524ba502ec2c66e0180712249d18b15d60" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-a5e5529309", "dataset": "supergpqa", "dataset_ref": "a5e5529309f946299aa9ffc6f8740613", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948396-HZFYzogd3TMctls4u1fE", "proof_sha256": "d7a68446a45b6fba3a925a54150e039c51b921a91b28cd6226e29b72643de4e0" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-ae0c184921", "dataset": "supergpqa", "dataset_ref": "ae0c184921f04375bb01feed6329b983", "category": "Engineering", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948407-i3lWgDYXyNgi7FnoDdgM", "proof_sha256": "80e84cc901e3f32792c8d9fcb29ce7e04d232fade79b2bcff875dc4a3607cd35" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-13ab6cdccf", "dataset": "supergpqa", "dataset_ref": "13ab6cdccf174f819230dc0c88ececf1", "category": "Engineering", "expected": "i", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948428-oQfhlwLqElqFt2qpeeU4", "proof_sha256": "eb2e6599168a0d79ee9aa04100648c9c6c656b57125cc34938f1439ba1b14793" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-c2fb6fe4a7", "dataset": "supergpqa", "dataset_ref": "c2fb6fe4a7e24551be8b4f182459e998", "category": "Engineering", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948443-mLIE2aQL1M5qEmW0VL7P", "proof_sha256": "0a9314a5e9c99310ccd754488db1cfdb1cb4c24232235a63bb8e0467e6b69605" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-88335aba2f", "dataset": "supergpqa", "dataset_ref": "88335aba2fb84a0daf9d7cc5b8dc603d", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948454-4cQPGo813sUoH7VTGKJZ", "proof_sha256": "285ce908fbee06e0ddc20162379cf7f5f6c9d92d048c43e0c6aff19ec58aee78" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-a4fb1c174d", "dataset": "supergpqa", "dataset_ref": "a4fb1c174d11471086e71dbf8715b46b", "category": "Engineering", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948468-1vGKTK74BalDRMTqILNJ", "proof_sha256": "b29fd6ebc401cec4c01161b56b516b1db9ba546482e569e8e55d07e9b30d6b35" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-fe80216e8d", "dataset": "supergpqa", "dataset_ref": "fe80216e8d484a3bb3dbcca289598c75", "category": "Engineering", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948491-EWFVQigXBRaDji0nGkPL", "proof_sha256": "d1fece49bbeecedc6c6f3aae306ec20589d819ac8a448dd39844af92efac9b13" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-e2a5fb96a9", "dataset": "supergpqa", "dataset_ref": "e2a5fb96a9a248da9f0b75b18bc0ac89", "category": "Engineering", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948511-ZqBbLOfNfhsbu1iWoSIZ", "proof_sha256": "4c7643497c8a16c3fa6dfa09dfa4315a46702c9e7e2394bc166f05f00f9550ab" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-07b7213c52", "dataset": "supergpqa", "dataset_ref": "07b7213c520e4e01a12010aff8cf899a", "category": "Engineering", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948521-PE6u6NR9C3dKe87GcJQv", "proof_sha256": "e2568b1f727ef2a79c3f3ffb5888daf3751d26664879d6669d6c486cc761a529" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-a7d50b09f7", "dataset": "supergpqa", "dataset_ref": "a7d50b09f7d2425bb92f62e716ef9421", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948530-BMmei0PepyWvHx2cMv7m", "proof_sha256": "18acf8316579f07f9ba51c620745555c3d0f31017365320db6aa656d0bb42307" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-cb1f3d75e5", "dataset": "supergpqa", "dataset_ref": "cb1f3d75e5a9402a9bb7755b6f6a029c", "category": "Medicine", "expected": "g", "extracted": "a", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948549-mhnCXuqWLYlgj6vQUxwx", "proof_sha256": "93d29ecd108fadbd2830ef0415fab85939b4032d8282190d41060c45a557a6c8" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-09fbf9b229", "dataset": "supergpqa", "dataset_ref": "09fbf9b2295642aa84cbb5a736da47c6", "category": "Medicine", "expected": "f", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948560-gpiBcnSmGeOCjpiri5Ic", "proof_sha256": "aa25863d65d02b4c6fcafa36c8751c497d3acd351cb5c041b6bff66fae7e635a" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-1a02bce824", "dataset": "supergpqa", "dataset_ref": "1a02bce824604cb2835ee25eacdc8f39", "category": "Medicine", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948581-Ivg0nQWdj197d2aJgpi6", "proof_sha256": "61dc8b5213225106f801378eac0fa7d4925a81ba275c4c30b2fc873e7519f7b2" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-fab25aad3a", "dataset": "supergpqa", "dataset_ref": "fab25aad3a89427aaf6cdffc0295360e", "category": "Medicine", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948602-Bc1w1oAzJ4DTfQ7d2Oaw", "proof_sha256": "b701e090fb0e4bd1c16625429ae59542f61d6903a94ad2a09d66dbd8e7d18c17" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-2da92a03d3", "dataset": "supergpqa", "dataset_ref": "2da92a03d32a413da6915868a4fa8796", "category": "Medicine", "expected": "c", "extracted": "j", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948623-z0qPkT1Cs4UDdNEfphw9", "proof_sha256": "fb0585878a2b0ee9f98f1f921c9a303747ff60eef5ac4cc861e87df49c6d1cbb" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-839fe51584", "dataset": "supergpqa", "dataset_ref": "839fe51584e14edda9cfc09aade2bb64", "category": "Medicine", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948634-7OXueFVrQXaTaYycTFX1", "proof_sha256": "5af1120e0c32bbea983df0ac7822e74c8713671d7465272fce0c1c204b2a97fc" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-9210801477", "dataset": "supergpqa", "dataset_ref": "92108014772645f692a2b3bbe764bfa8", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948643-hdwGpUW8dahw1SzNNnEr", "proof_sha256": "0099c9b2a263965d76863d72fe004b6c63f5139b36b1cb7fe5748acf8ec2be78" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-236fa24710", "dataset": "supergpqa", "dataset_ref": "236fa247100e457c87371ea445f9aaea", "category": "Medicine", "expected": "f", "extracted": "d", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948664-GJiJI5wy4jXbEsRrUOM6", "proof_sha256": "9b4991517ee8cbf1f0661822a9e4ae0beb590000d922ccdbe1234e4ac583fde3" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-c134c966d6", "dataset": "supergpqa", "dataset_ref": "c134c966d6d04da0b1d6b1a268a70744", "category": "Economics", "expected": "b", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948682-MwmzE0Wu2nqUcLyU6rEP", "proof_sha256": "3657bee08ec123f207f4c621eb710d7c825b8a3b91007a7c840f83704cf43927" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-3d08528054", "dataset": "supergpqa", "dataset_ref": "3d085280542d48c0b14750a3d930afed", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948700-lsI5xznTx6cbA1VMDqnf", "proof_sha256": "b391cfe010b15aea9f49b788a6f2fb6623dacf4e359d83db347c1ecc4cb6ba3b" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-d7c9230658", "dataset": "supergpqa", "dataset_ref": "d7c9230658bf464b8d6fa4ce25a6dad6", "category": "Economics", "expected": "f", "extracted": "f", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948710-SryaJlImZyHqqD6ymG7U", "proof_sha256": "16aa66649e794ee7c7c12379fdac2164d4f1efd3c751f3e3a2aeff90c9559700" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-9115b55ac6", "dataset": "supergpqa", "dataset_ref": "9115b55ac6af4a83a190638609929ecc", "category": "Economics", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948719-4Ji2162RrcfTxlLC7Far", "proof_sha256": "d74cca4d8a1a3bb747d937915a6351bc0dba01d73ea7a73f1d4e24d5c2fe48ab" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-dd5c689dda", "dataset": "supergpqa", "dataset_ref": "dd5c689ddaa44801ab257eeb343ec9fb", "category": "Law", "expected": "d", "extracted": "b", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948742-tjM8Ec8yd6PbWL2mWpeH", "proof_sha256": "679a786aac36a03a373bd3d47d9b3ed9572e7a9c607abc252d08ab52dfce90d3" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "sgpqa-1ad8fa613e", "dataset": "supergpqa", "dataset_ref": "1ad8fa613eb240b69f718e4035dc6ae9", "category": "Law", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948764-MCS34IDEdYrpQOvl2wqD", "proof_sha256": "fc204c8f1e916586ceedc3b5f005c4ffdf7687420d3afafb004152427dd5c3c2" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "mmlupro-1103", "dataset": "mmlu-pro", "dataset_ref": "1103", "category": "Law", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948774-DAAU1HRtO7fscZItHFon", "proof_sha256": "1178707ea7bea67b1fc66b3bdca3c5e0e652ebb71ab58dc35fd19dd78f2bfe07" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "mmlupro-1129", "dataset": "mmlu-pro", "dataset_ref": "1129", "category": "Law", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948788-IsGyFHWk8WgM9dGMSx1L", "proof_sha256": "cf7495112f028dae144ca48dd1b6ba6fc48116bf5eab329c8775e2cfcb95d053" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "mmlupro-1110", "dataset": "mmlu-pro", "dataset_ref": "1110", "category": "Law", "expected": "i", "extracted": "c", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948798-P3z75Xp9zUPFrunFtA6Z", "proof_sha256": "c82db9fe8507e966c8a2f5ef917460016c82cb0e0dc5e0fee61ca6028998a026" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "mmlupro-1174", "dataset": "mmlu-pro", "dataset_ref": "1174", "category": "Law", "expected": "b", "extracted": "h", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948809-wyMXbRkwQ5y708lezZNC", "proof_sha256": "4b7418b3ed74a5fc6ebf04df147e7987fe498ae8d1279dfe73aa7e6c391fc5f4" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "mmlupro-11264", "dataset": "mmlu-pro", "dataset_ref": "11264", "category": "Humanities & social science", "expected": "e", "extracted": "g", "correct": false, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948819-Jj5mGmNY9RZpLZzGASTt", "proof_sha256": "f6eefbb31641013c748c98036fb3369f0bd0e013c5e3b1095d7b7f4504d56778" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "mmlupro-11281", "dataset": "mmlu-pro", "dataset_ref": "11281", "category": "Humanities & social science", "expected": "a", "extracted": "a", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948831-872Y3fmEUVPJ3e0lKyZz", "proof_sha256": "f46c5420456fe0f5f5121750d100262d80bc6bb15758af3486f7a0273f0a22d2" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "mmlupro-11228", "dataset": "mmlu-pro", "dataset_ref": "11228", "category": "Humanities & social science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948842-miNsfvp1iS9ggaTzIwBL", "proof_sha256": "38ab1d2d397725ccf17cce1a105baf876fd9cd78a9c5162c4fbb82eaa4ba93e1" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "mmlupro-2102", "dataset": "mmlu-pro", "dataset_ref": "2102", "category": "Humanities & social science", "expected": "b", "extracted": "b", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948852-wxDEQCmb8pBqrOtnPyfL", "proof_sha256": "fa603d1b5b07dd3fa45557ca4006ff883422cb13cdfd210c0b67346d883baedb" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "mmlupro-2163", "dataset": "mmlu-pro", "dataset_ref": "2163", "category": "Humanities & social science", "expected": "h", "extracted": "h", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948861-euMhLFmblxtk99KyJAG9", "proof_sha256": "4f80e23fc625b209fe4dca6da546af21d078210d6054728b453f66f05ff57e9f" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "mmlupro-2187", "dataset": "mmlu-pro", "dataset_ref": "2187", "category": "Humanities & social science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948880-sQGo3CTm2Wz0OuuZbCqI", "proof_sha256": "d6908ce73a95de23f7afc0976ae12599901f98c9abf7f413baf457d5aa29afa3" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "mmlupro-5199", "dataset": "mmlu-pro", "dataset_ref": "5199", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948892-6gbYeVOoI7zYstvncRDI", "proof_sha256": "a3acd4f1b04724b12b44726b53d6021884140fe8d264a2059d4cac3a16a341d8" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "mmlupro-5196", "dataset": "mmlu-pro", "dataset_ref": "5196", "category": "Humanities & social science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948902-ajk7J2xQ6AJxka1ykUvE", "proof_sha256": "856b245e6f08ace82579c808da3d6163b5384c323253c8bbe572bace49641cf8" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "mmlupro-5197", "dataset": "mmlu-pro", "dataset_ref": "5197", "category": "Humanities & social science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948919-BkNRQXqp2lMUVNwpZlD8", "proof_sha256": "ac1e4790e81c0e81d3e72f0791baf2c7e6aabf3d16c72f4e983791d6b24e7a42" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "mmlupro-95", "dataset": "mmlu-pro", "dataset_ref": "95", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948938-i8c193KjcfXwLmZlAlp0", "proof_sha256": "913bdfd705d862d21fcc5c52795d20de619085ee72e81627c5a2eda2379d4f06" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "mmlupro-155", "dataset": "mmlu-pro", "dataset_ref": "155", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948947-8S6JMlOnXopOiJnrMqbf", "proof_sha256": "5f0bf43544093f17df0e7888f16d586a1ae365c49a7228d4902b83c16bdc2369" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "mmlupro-150", "dataset": "mmlu-pro", "dataset_ref": "150", "category": "Economics", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948955-petdsbQAeiiFlP1yGljA", "proof_sha256": "8ec1b6cdf6461ba68f2e5169859fed0e44ae9c49095e235cc8866b7f71df38d5" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "mmlupro-6208", "dataset": "mmlu-pro", "dataset_ref": "6208", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948972-x9QSC9lnpJjwTpOUU3aS", "proof_sha256": "ffd5379f1c88008543ed443264b9998ff45e8f8fa603c87cc4e6ee4425c6618c" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "mmlupro-6247", "dataset": "mmlu-pro", "dataset_ref": "6247", "category": "Medicine", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792948983-tFlyf9GL1f5bXVCf1XF0", "proof_sha256": "5336752f16cab218164306bb750c9d293cdabf6b1968d3206b5e84b73114d155" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "mmlupro-6218", "dataset": "mmlu-pro", "dataset_ref": "6218", "category": "Medicine", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792949002-W6I4fi1iKo2BJYOzv0ri", "proof_sha256": "e99c6ec3f053700bab6334765e7a3d3f37c5976a46040c9b79de1ff7897044f3" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "mmlupro-7236", "dataset": "mmlu-pro", "dataset_ref": "7236", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792949021-XxmRmPKWRXExnPCvssz7", "proof_sha256": "2208353689a190f615d6995c77fde3758d94424a710d2383d450ee3a0a8efa50" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "mmlupro-7181", "dataset": "mmlu-pro", "dataset_ref": "7181", "category": "Economics", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792949030-I4KYYqjFY2T3mxej9QIh", "proof_sha256": "0b7c8783b1d33290c30aade3e39a6fca2ae39bea1d65cc592c6cd9500888cea3" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "mmlupro-8212", "dataset": "mmlu-pro", "dataset_ref": "8212", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792949048-VvRcnf41h1x0FFtyypEq", "proof_sha256": "2fc78c020b574d38e234efab72fb78f4431d7fb69d18c4d3161c5e940b2fa333" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "mmlupro-8232", "dataset": "mmlu-pro", "dataset_ref": "8232", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792949068-ZAvxT7ahjMZbtiKGU6d0", "proof_sha256": "6b9d549a676d36508a5295cd99eb73244dfe8e79ee3408ff230a6644f5b3b6f6" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "mmlupro-10231", "dataset": "mmlu-pro", "dataset_ref": "10231", "category": "Science", "expected": "i", "extracted": "i", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792949086-6wYxBXgh5DiqKZSbekza", "proof_sha256": "f3d457fcc2b3d10d932055107f394ef7afdbcc3fbeb43296bfe855f25ed0369d" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "mmlupro-10264", "dataset": "mmlu-pro", "dataset_ref": "10264", "category": "Science", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792949095-KkZlz3ceXatSwohfxvwd", "proof_sha256": "b866f940709d0a145e1f2af448cdb22184788efd7d7e1c0b98d7449b17d8cae6" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "mmlupro-4153", "dataset": "mmlu-pro", "dataset_ref": "4153", "category": "Science", "expected": "c", "extracted": "c", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792949105-EydAKvGFZkuoJIokauec", "proof_sha256": "cebe53f0185cdfddf68d44c975084e964d9c8b2642f1ca2256d2957afe44e43f" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "mmlupro-4193", "dataset": "mmlu-pro", "dataset_ref": "4193", "category": "Science", "expected": "d", "extracted": "d", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792949124-sEuSdQ8cqUUCRlmvFCqr", "proof_sha256": "871a2fb5c2a35b0ec7bdc70a7e19fe728453cca46b7b7bb261fb82fde4849ae3" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "mmlupro-3138", "dataset": "mmlu-pro", "dataset_ref": "3138", "category": "Science", "expected": "e", "extracted": "e", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792949134-oojzSCr6sml4rPpmss7t", "proof_sha256": "1b28fc06b86976cae2f79804948f055a6cb3e7181e7b82179549782db791e437" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "mmlupro-3177", "dataset": "mmlu-pro", "dataset_ref": "3177", "category": "Science", "expected": "g", "extracted": "g", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792949152-COqQ2RCQBom2ttcd4Xzj", "proof_sha256": "f4bd301bd8be727d6ed97c8b2ca110ea8ee58af35fa24e69e80623498d207c5a" }, { "model": "anthropic/claude-sonnet-5.5", "item_id": "mmlupro-11314", "dataset": "mmlu-pro", "dataset_ref": "11314", "category": "Engineering", "expected": "j", "extracted": "j", "correct": true, "errored": false, "refused": false, "excluded_not_in_manifest": false, "finish_reason": "stop", "host": "Claude Platform on AWS", "run_id": "20261025T170652Z", "generation_id": "gen-1792949162-AV7aAcZxMiv7WlIfbupi", "proof_sha256": "b1f52d2c2839e36c371dbb033f1ce10d3f57b5c5d61d6ad2cdcaa5a5fa95c79e" } ] }